def contest_import_from_dict(cls, data): """Build the object using data from a dictionary. """ data['tasks'] = [Task.import_from_dict(task_data) for task_data in data['tasks']] tasks_by_name = dict(map(lambda x: (x.name, x), data['tasks'])) data['users'] = [User.import_from_dict(user_data, tasks_by_name=tasks_by_name) for user_data in data['users']] data['announcements'] = [Announcement.import_from_dict(ann_data) for ann_data in data['announcements']] if 'start' in data and data['start'] is not None: data['start'] = make_datetime(data['start']) if 'stop' in data and data['stop'] is not None: data['stop'] = make_datetime(data['stop']) if 'token_min_interval' in data: data['token_min_interval'] = \ timedelta(seconds=data['token_min_interval']) if 'token_gen_time' in data: data['token_gen_time'] = timedelta(minutes=data['token_gen_time']) if 'per_user_time' in data and data['per_user_time'] is not None: data['per_user_time'] = timedelta(seconds=data['per_user_time']) if 'min_submission_interval' in data and \ data['min_submission_interval'] is not None: data['min_submission_interval'] = \ timedelta(seconds=data['min_submission_interval']) if 'min_usertest_interval' in data and \ data['min_usertest_interval'] is not None: data['min_usertest_interval'] = \ timedelta(seconds=data['min_usertest_interval']) return cls(**data)
def import_from_dict(cls, data): """Build the object using data from a dictionary. """ data['question_timestamp'] = make_datetime(data['question_timestamp']) if data['reply_timestamp'] is not None: data['reply_timestamp'] = make_datetime(data['reply_timestamp']) return cls(**data)
def user_import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data['messages'] = [ Message.import_from_dict(message_data) for message_data in data['messages'] ] data['questions'] = [ Question.import_from_dict(question_data) for question_data in data['questions'] ] data['submissions'] = [ Submission.import_from_dict(submission_data, tasks_by_name=tasks_by_name) for submission_data in data['submissions'] ] data['user_tests'] = [ UserTest.import_from_dict(user_test_data, tasks_by_name=tasks_by_name) for user_test_data in data['user_tests'] ] if 'starting_time' in data and data['starting_time'] is not None: data['starting_time'] = make_datetime(data['starting_time']) if 'extra_time' in data: data['extra_time'] = timedelta(seconds=data['extra_time']) obj = cls(**data) for submission in obj.submissions: submission.user = obj return obj
def import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data['files'] = [ File.import_from_dict(file_data) for file_data in data['files'] ] data['files'] = dict([(_file.filename, _file) for _file in data['files']]) data['executables'] = [ Executable.import_from_dict(executable_data) for executable_data in data['executables'] ] data['executables'] = dict([(executable.filename, executable) for executable in data['executables']]) data['evaluations'] = [ Evaluation.import_from_dict(eval_data) for eval_data in data['evaluations'] ] if data['token'] is not None: data['token'] = Token.import_from_dict(data['token']) data['task'] = tasks_by_name[data['task']] data['user'] = None data['timestamp'] = make_datetime(data['timestamp']) return cls(**data)
def import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data['files'] = [ UserTestFile.import_from_dict(file_data) for file_data in data['files'] ] data['files'] = dict([(_file.filename, _file) for _file in data['files']]) data['executables'] = [ UserTestExecutable.import_from_dict(executable_data) for executable_data in data['executables'] ] data['executables'] = dict([(executable.filename, executable) for executable in data['executables']]) data['managers'] = [ UserTestManager.import_from_dict(manager_data) for manager_data in data['managers'] ] data['managers'] = dict([(manager.filename, manager) for manager in data['managers']]) data['task'] = tasks_by_name[data['task']] data['user'] = None data['timestamp'] = make_datetime(data['timestamp']) return cls(**data)
def acquire_worker(self, job, side_data=None): """Tries to assign a job to an available worker. If no workers are available then this returns None, otherwise this returns the chosen worker. job (job): the job to assign to a worker side_data (object): object to attach to the worker for later use returns (int): None if no workers are available, the worker assigned to the job otherwise """ # We look for an available worker try: shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True) except LookupError: return None # Then we fill the info for future memory self._job[shard] = job self._start_time[shard] = make_datetime() self._side_data[shard] = side_data logger.debug("Worker %s acquired." % shard) # And finally we ask the worker to do the job action, object_id = job timestamp = side_data[1] queue_time = self._start_time[shard] - timestamp logger.info( "Asking worker %s to %s submission/user test %d " " (%s after submission)." % (shard, action, object_id, queue_time) ) with SessionGen(commit=False) as session: if action == EvaluationService.JOB_TYPE_COMPILATION: submission = Submission.get_from_id(object_id, session) job_ = CompilationJob.from_submission(submission) elif action == EvaluationService.JOB_TYPE_EVALUATION: submission = Submission.get_from_id(object_id, session) job_ = EvaluationJob.from_submission(submission) elif action == EvaluationService.JOB_TYPE_TEST_COMPILATION: user_test = UserTest.get_from_id(object_id, session) job_ = CompilationJob.from_user_test(user_test) elif action == EvaluationService.JOB_TYPE_TEST_EVALUATION: user_test = UserTest.get_from_id(object_id, session) job_ = EvaluationJob.from_user_test(user_test) job_.get_output = True job_.only_execution = True self._worker[shard].execute_job( job_dict=job_.export_to_dict(), callback=self._service.action_finished.im_func, plus=(action, object_id, side_data, shard), ) return shard
def import_object(self, data): """Import objects from the given data (without relationships). The given data is assumed to be a dict in the format produced by ContestExporter. This method reads the "_class" item and tries to find the corresponding class. Then it loads all column properties of that class (those that are present in the data) and uses them as keyword arguments in a call to the class constructor (if a required property is missing this call will raise an error). Relationships are not handled by this method, since we may not have all referenced objects available yet. Thus we prefer to add relationships in a later moment, using the add_relationships method. Note that both this method and add_relationships don't check if the given data has more items than the ones we understand and use. """ cls = getattr(class_hook, data["_class"]) args = dict() for prp in cls._col_props: if prp.key not in data: # We will let the __init__ of the class check if any # argument is missing, so it's safe to just skip here. continue col = prp.columns[0] col_type = type(col.type) val = data[prp.key] if col_type in [Boolean, Integer, Float, Unicode, RepeatedUnicode]: args[prp.key] = val elif col_type is String: args[prp.key] = val.encode( 'latin1') if val is not None else None elif col_type is DateTime: args[prp.key] = make_datetime(val) if val is not None else None elif col_type is Interval: args[prp.key] = timedelta( seconds=val) if val is not None else None else: raise RuntimeError("Unknown SQLAlchemy column type: %s" % col_type) return cls(**args)
def import_object(self, data): """Import objects from the given data (without relationships). The given data is assumed to be a dict in the format produced by ContestExporter. This method reads the "_class" item and tries to find the corresponding class. Then it loads all column properties of that class (those that are present in the data) and uses them as keyword arguments in a call to the class constructor (if a required property is missing this call will raise an error). Relationships are not handled by this method, since we may not have all referenced objects available yet. Thus we prefer to add relationships in a later moment, using the add_relationships method. Note that both this method and add_relationships don't check if the given data has more items than the ones we understand and use. """ cls = getattr(class_hook, data["_class"]) args = dict() for prp in cls._col_props: if prp.key not in data: # We will let the __init__ of the class check if any # argument is missing, so it's safe to just skip here. continue col = prp.columns[0] col_type = type(col.type) val = data[prp.key] if col_type in [Boolean, Integer, Float, Unicode, RepeatedUnicode]: args[prp.key] = val elif col_type is String: args[prp.key] = val.encode('latin1') if val is not None else None elif col_type is DateTime: args[prp.key] = make_datetime(val) if val is not None else None elif col_type is Interval: args[prp.key] = timedelta(seconds=val) if val is not None else None else: raise RuntimeError("Unknown SQLAlchemy column type: %s" % col_type) return cls(**args)
def push(self, job, priority, timestamp=None): """Push a job in the queue. If timestamp is not specified, uses the current time. job (job): a couple (job_type, submission_id) priority (int): the priority of the job timestamp (int): the time of the submission """ if timestamp is None: timestamp = make_datetime() self._queue.append((priority, timestamp, job)) last = len(self._queue) - 1 self._reverse[job] = last self._up_heap(last)
def import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data["files"] = [UserTestFile.import_from_dict(file_data) for file_data in data["files"]] data["files"] = dict([(_file.filename, _file) for _file in data["files"]]) data["executables"] = [ UserTestExecutable.import_from_dict(executable_data) for executable_data in data["executables"] ] data["executables"] = dict([(executable.filename, executable) for executable in data["executables"]]) data["managers"] = [UserTestManager.import_from_dict(manager_data) for manager_data in data["managers"]] data["managers"] = dict([(manager.filename, manager) for manager in data["managers"]]) data["task"] = tasks_by_name[data["task"]] data["user"] = None data["timestamp"] = make_datetime(data["timestamp"]) return cls(**data)
def import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data['files'] = [File.import_from_dict(file_data) for file_data in data['files']] data['files'] = dict([(_file.filename, _file) for _file in data['files']]) if data['token'] is not None: data['token'] = Token.import_from_dict(data['token']) data['task'] = tasks_by_name[data['task']] data['user'] = None data['timestamp'] = make_datetime(data['timestamp']) data['results'] = [SubmissionResult.import_from_dict(_r, data['task']) for _r in data['results']] data['results'] = dict([(_r.dataset_id, _r) for _r in data['results']]) return cls(**data)
def format_datetime_smart(dt, timezone, locale=None): """Return dt formatted as 'date & time' or, if date is today, just 'time' dt (datetime): a datetime object timezone (subclass of tzinfo): the timezone the output should be in return (str): the [date and] time of dt, formatted using the given locale """ if locale is None: locale = tornado.locale.get() # convert dt and 'now' from UTC to local time dt = dt.replace(tzinfo=utc).astimezone(timezone) now = make_datetime().replace(tzinfo=utc).astimezone(timezone) if dt.date() == now.date(): return dt.strftime(locale.translate("%H:%M:%S")) else: return dt.strftime(locale.translate("%Y-%m-%d %H:%M:%S"))
def user_import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data['messages'] = [Message.import_from_dict(message_data) for message_data in data['messages']] data['questions'] = [Question.import_from_dict(question_data) for question_data in data['questions']] data['submissions'] = [Submission.import_from_dict( submission_data, tasks_by_name=tasks_by_name) for submission_data in data['submissions']] if 'starting_time' in data and data['starting_time'] is not None: data['starting_time'] = make_datetime(data['starting_time']) if 'extra_time' in data: data['extra_time'] = timedelta(seconds=data['extra_time']) obj = cls(**data) for submission in obj.submissions: submission.user = obj return obj
def import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data['files'] = [ File.import_from_dict(file_data) for file_data in data['files'] ] data['files'] = dict([(_file.filename, _file) for _file in data['files']]) if data['token'] is not None: data['token'] = Token.import_from_dict(data['token']) data['task'] = tasks_by_name[data['task']] data['user'] = None data['timestamp'] = make_datetime(data['timestamp']) data['results'] = [ SubmissionResult.import_from_dict(_r, data['task']) for _r in data['results'] ] data['results'] = dict([(_r.dataset_id, _r) for _r in data['results']]) return cls(**data)
def import_from_dict(cls, data, tasks_by_name): """Build the object using data from a dictionary. """ data['files'] = [File.import_from_dict(file_data) for file_data in data['files']] data['files'] = dict([(_file.filename, _file) for _file in data['files']]) data['executables'] = [Executable.import_from_dict(executable_data) for executable_data in data['executables']] data['executables'] = dict([(executable.filename, executable) for executable in data['executables']]) data['evaluations'] = [Evaluation.import_from_dict(eval_data) for eval_data in data['evaluations']] if data['token'] is not None: data['token'] = Token.import_from_dict(data['token']) data['task'] = tasks_by_name[data['task']] data['user'] = None data['timestamp'] = make_datetime(data['timestamp']) return cls(**data)
def check_timeouts(self): """Check if some worker is not responding in too much time. If this is the case, the worker is scheduled for disabling, and we send him a message trying to shut it down. return (list): list of tuples (priority, timestamp, job) of jobs assigned to worker that timeout. """ now = make_datetime() lost_jobs = [] for shard in self._worker: if self._start_time[shard] is not None: active_for = now - self._start_time[shard] if active_for > EvaluationService.WORKER_TIMEOUT: # Here shard is a working worker with no sign of # intelligent life for too much time. logger.error( "Disabling and shutting down " "worker %d because of no reponse " "in %s." % (shard, active_for) ) assert ( self._job[shard] != WorkerPool.WORKER_INACTIVE and self._job[shard] != WorkerPool.WORKER_DISABLED ) # We return the job so ES can do what it needs. if not self._ignore[shard]: job = self._job[shard] priority, timestamp = self._side_data[shard] lost_jobs.append((priority, timestamp, job)) # Also, we are not trusting it, so we are not # assigning him new jobs even if it comes back to # life. self._schedule_disabling[shard] = True self._ignore[shard] = True self.release_worker(shard) self._worker[shard].quit("No response in %s." % active_for) return lost_jobs
def import_from_dict(cls, data): """Build the object using data from a dictionary. """ data['timestamp'] = make_datetime(data['timestamp']) return cls(**data)
def tokens_available(self, username, task_name, timestamp=None): """Return three pieces of data: [0] the number of available tokens for the user to play on the task (independently from the fact that (s)he can play it right now or not due to a min_interval wating for expiration); -1 means infinite tokens; [1] the next time in which a token will be generated (or None); from the user perspective, i.e.: if the user will do nothing, [1] is the first time in which his number of available tokens will be greater than [0]; [2] the time when the min_interval will expire, or None In particular, let r the return value of this method. We can sketch the code in the following way.: if r[0] > 0 or r[0] == -1: we have tokens if r[2] is None: we can play a token else: we must wait till r[2] to play a token if r[1] is not None: next one will be generated at r[1] else: no other tokens will be generated (max/total reached ?) else: we don't have tokens right now if r[1] is not None: next one will be generated at r[1] if r[2] is not None and r[2] > r[1]: but we must wait also till r[2] to play it else: no other tokens will be generated (max/total reached ?) Note also that this method assumes that all played tokens were regularly played, and that there are no tokens played in the future. Also, if r[0] == 0 and r[1] is None, then r[2] should be ignored. username (string): the username of the user. task_name (string): the name of the task. timestamp (int): the time relative to which making the calculation return ((int, int, int)): see description above the latter two are timestamps, or None. """ if timestamp is None: timestamp = make_datetime() user = self.get_user(username) task = self.get_task(task_name) # Take the list of the tokens already played (sorted by time). tokens = user.get_tokens() token_timestamps_contest = sorted([token.timestamp for token in tokens]) token_timestamps_task = sorted([ token.timestamp for token in tokens if token.submission.task.name == task_name]) # If the contest is USACO-style (i.e., the time for each user # start when he/she logs in for the first time), then we start # accumulating tokens from the user starting time; otherwise, # from the start of the contest. start = self.start if self.per_user_time is not None: start = user.starting_time # Compute separately for contest-wise and task-wise. res_contest = Contest._tokens_available( token_timestamps_contest, self.token_initial, self.token_max, self.token_total, self.token_min_interval, self.token_gen_time, self.token_gen_number, start, timestamp) res_task = Contest._tokens_available( token_timestamps_task, task.token_initial, task.token_max, task.token_total, task.token_min_interval, task.token_gen_time, task.token_gen_number, start, timestamp) # Merge the results. # First, the "expiration". if res_contest[2] is None: expiration = res_task[2] elif res_task[2] is None: expiration = res_contest[2] else: expiration = max(res_task[2], res_contest[2]) # Then, check if both are infinite if res_contest[0] == -1 and res_task[0] == -1: res = (-1, None, expiration) # Else, "combine" them appropriately. else: # Having infinite contest tokens, in this situation, is the # same as having a finite number that is strictly greater # than the task tokens. The same holds the other way, too. if res_contest[0] == -1: res_contest = (res_task[0] + 1, None, None) if res_task[0] == -1: res_task = (res_contest[0] + 1, None, None) # About next token generation time: we need to see when the # *minimum* between res_contest[0] and res_task[0] is # increased by one, so if there is an actual minimum we # need to consider only the next generation time for it. # Otherwise, if they are equal, we need both to generate an # additional token and we store the maximum between the two # next times of generation. if res_contest[0] < res_task[0]: # We have more task-tokens than contest-tokens. # We just need a contest-token to be generated. res = (res_contest[0], res_contest[1], expiration) elif res_task[0] < res_contest[0]: # We have more contest-tokens than task-tokens. # We just need a task-token to be generated. res = (res_task[0], res_task[1], expiration) else: # Darn, we need both! if res_contest[1] is None or res_task[1] is None: res = (res_task[0], None, expiration) else: res = (res_task[0], max(res_contest[1], res_task[1]), expiration) return res
def __init__(self, timestamp=None, submission=None): if timestamp is None: timestamp = make_datetime() self.timestamp = timestamp self.submission = submission
# The rules describing how to encode/decode column properties to/from # JSON. They're indexed by the SQLAlchemy column type and their values # are in the form (python type(s), decoder, encoder). _TYPE_MAP = { Boolean: (bool, lambda x: x, lambda x: x), Integer: (six.integer_types, lambda x: x, lambda x: x), Float: (six.integer_types + (float,), lambda x: x, lambda x: x), String: (six.text_type, lambda x: x.encode('latin1'), lambda x: x.decode('latin1')), Unicode: (six.text_type, lambda x: x, lambda x: x), DateTime: (six.integer_types + (float,), lambda x: make_datetime(x), lambda x: make_timestamp(x)), Interval: (six.integer_types + (float,), lambda x: timedelta(seconds=x), lambda x: x.total_seconds()), } # All the URLs we need for each database class, as a template. URLs are # the one described in the module-level docstring, whereas endpoints # are in the form "<class name>/<action>". _ENTITY_TEMPLATE = RuleTemplate([ EndpointPrefix('$name/', [ Submount('/$path', [ Rule("/", methods=["GET"], endpoint="list"), Rule("/", methods=["POST"], endpoint="create"), Rule("/$pkey", methods=["GET"], endpoint="retrieve"), Rule("/$pkey/<prp_key>", methods=["GET"], endpoint="sublist"),