def get_aws_browser():
global aws_browser
if aws_browser is None:
aws_browser = Browser()
lr = AWSLoginRequest(aws_browser,
admin_info["username"], admin_info["password"],
base_url=AWS_BASE_URL)
aws_browser.login(lr)
return aws_browser
def get_cws_browser(user_id):
global cws_browser
if cws_browser is None:
cws_browser = Browser()
username = created_users[user_id]['username']
password = created_users[user_id]['password']
lr = CWSLoginRequest(
cws_browser, username, password, base_url=CWS_BASE_URL)
cws_browser.login(lr)
return cws_browser
def get_cws_browser(self, user_id):
if self._cws_browser is None:
self._cws_browser = Browser()
username = self.created_users[user_id]['username']
password = self.created_users[user_id]['password']
lr = CWSLoginRequest(
self._cws_browser, username, password,
base_url=self.CWS_BASE_URL)
self._cws_browser.login(lr)
return self._cws_browser
def get_aws_browser(self):
if self._aws_browser is None:
self._aws_browser = Browser()
lr = AWSLoginRequest(self._aws_browser,
self.admin_info["username"],
self.admin_info["password"],
base_url=self.AWS_BASE_URL)
self._aws_browser.login(lr)
return self._aws_browser
def get_cws_browser(user_id):
global cws_browser
if cws_browser is None:
cws_browser = Browser()
username = created_users[user_id]['username']
password = created_users[user_id]['password']
lr = CWSLoginRequest(
cws_browser, username, password, base_url=CWS_BASE_URL)
cws_browser.login(lr)
return cws_browser
def get_aws_browser():
global aws_browser
if aws_browser is None:
aws_browser = Browser()
lr = AWSLoginRequest(aws_browser,
admin_info["username"], admin_info["password"],
base_url=AWS_BASE_URL)
aws_browser.login(lr)
return aws_browser
def __init__(self, username, password, metrics, tasks,
log=None, base_url=None, submissions_path=None):
threading.Thread.__init__(self)
self.username = username
self.password = password
self.metrics = metrics
self.tasks = tasks
self.log = log
self.base_url = base_url
self.submissions_path = submissions_path
self.name = "Actor thread for user %s" % (self.username)
self.browser = Browser()
self.die = False
def __init__(self, username, password, metrics, tasks,
log=None, base_url=None, submissions_path=None):
threading.Thread.__init__(self)
self.username = username
self.password = password
self.metrics = metrics
self.tasks = tasks
self.log = log
self.base_url = base_url
self.submissions_path = submissions_path
self.name = "Actor thread for user %s" % (self.username)
self.browser = Browser()
self.die = False
class Actor(threading.Thread):
"""Class that simulates the behaviour of a user of the system. It
performs some requests at randomized times (checking CMS pages,
doing submissions, ...), checking for their success or failure.
The probability that the users doing actions depends on the value
specified in an object called "metrics".
"""
def __init__(self,
username,
password,
metrics,
tasks,
log=None,
base_url=None,
submissions_path=None):
threading.Thread.__init__(self)
self.username = username
self.password = password
self.metrics = metrics
self.tasks = tasks
self.log = log
self.base_url = base_url
self.submissions_path = submissions_path
self.name = "Actor thread for user %s" % (self.username)
self.browser = Browser()
self.die = False
def run(self):
try:
print("Starting actor for user %s" % (self.username),
file=sys.stderr)
self.act()
except ActorDying:
print("Actor dying for user %s" % (self.username), file=sys.stderr)
def act(self):
"""Define the behaviour of the actor. Subclasses are expected
to overwrite this stub method properly.
"""
raise Exception("Not implemented. Please subclass Action"
"and overwrite act().")
def do_step(self, request):
self.wait_next()
self.log.total += 1
try:
request.execute()
except Exception as exc:
print("Unhandled exception while executing the request: %s" % exc,
file=sys.stderr)
return
self.log.__dict__[request.outcome] += 1
self.log.total_time += request.duration
self.log.max_time = max(self.log.max_time, request.duration)
self.log.store_to_file(request)
def wait_next(self):
"""Wait some time. At the moment it waits c*X seconds, where c
is the time_coeff parameter in metrics and X is an
exponentially distributed random variable, with parameter
time_lambda in metrics.
The total waiting time is divided in lots of little sleep()
call each one of 0.1 seconds, so that the waiting gets
interrupted if a die signal arrives.
If a die signal is received, an ActorDying exception is
raised.
"""
SLEEP_PERIOD = 0.1
time_to_wait = self.metrics['time_coeff'] * \
random.expovariate(self.metrics['time_lambda'])
sleep_num = time_to_wait // SLEEP_PERIOD
remaining_sleep = time_to_wait - (sleep_num * SLEEP_PERIOD)
for _ in range(sleep_num):
time.sleep(SLEEP_PERIOD)
if self.die:
raise ActorDying()
time.sleep(remaining_sleep)
if self.die:
raise ActorDying()
def login(self):
"""Log in and check to be logged in."""
self.do_step(
HomepageRequest(self.browser,
self.username,
loggedin=False,
base_url=self.base_url))
lr = CWSLoginRequest(self.browser,
self.username,
self.password,
base_url=self.base_url)
self.browser.read_xsrf_token(lr.base_url)
self.do_step(lr)
self.do_step(
HomepageRequest(self.browser,
self.username,
loggedin=True,
base_url=self.base_url))
class FunctionalTestFramework(object):
"""An object encapsulating the status of a functional test
It maintains facilities to interact with the services while running a
functional tests, e.g. via virtual browsers, and also offers facilities
to create and retrieve objects from the services.
"""
# Base URLs for AWS and CWS
AWS_BASE_URL = "http://localhost:8889"
CWS_BASE_URL = "http://localhost:8888"
# Regexes for submission statuses.
WAITING_STATUSES = re.compile(
r'Compiling\.\.\.|Evaluating\.\.\.|Scoring\.\.\.|Evaluated')
COMPLETED_STATUSES = re.compile(
r'Compilation failed|Evaluated \(|Scored \(')
# Regexes for user test statuses
WAITING_STATUSES_USER_TEST = re.compile(
r'Compiling\.\.\.|Evaluating\.\.\.')
COMPLETED_STATUSES_USER_TEST = re.compile(
r'Compilation failed|Evaluated')
# Singleton instance for this class.
__instance = None
def __new__(cls):
if FunctionalTestFramework.__instance is None:
FunctionalTestFramework.__instance = object.__new__(cls)
return FunctionalTestFramework.__instance
def __init__(self):
# This holds the decoded-JSON of the cms.conf configuration file.
# Lazily loaded, to be accessed through the getter method.
self._cms_config = None
# Persistent browsers to access AWS and CWS. Lazily loaded, to be
# accessed through the getter methods.
self._aws_browser = None
self._cws_browser = None
# List of users and tasks we created as part of the test.
self.created_users = {}
self.created_tasks = {}
# Information on the administrator running the tests.
self.admin_info = {}
def get_aws_browser(self):
if self._aws_browser is None:
self._aws_browser = Browser()
lr = AWSLoginRequest(self._aws_browser,
self.admin_info["username"],
self.admin_info["password"],
base_url=self.AWS_BASE_URL)
self._aws_browser.login(lr)
return self._aws_browser
def get_cws_browser(self, user_id):
if self._cws_browser is None:
self._cws_browser = Browser()
username = self.created_users[user_id]['username']
password = self.created_users[user_id]['password']
lr = CWSLoginRequest(
self._cws_browser, username, password,
base_url=self.CWS_BASE_URL)
self._cws_browser.login(lr)
return self._cws_browser
def initialize_aws(self):
"""Create an admin.
The username will be admin_<suffix>, where <suffix> will be the first
integer (from 1) for which an admin with that name doesn't yet exist.
return (str): the suffix.
"""
logger.info("Creating admin...")
self.admin_info["password"] = "******"
suffix = "1"
while True:
self.admin_info["username"] = "******" % suffix
logger.info("Trying %(username)s" % self.admin_info)
try:
sh([sys.executable, "cmscontrib/AddAdmin.py",
"%(username)s" % self.admin_info,
"-p", "%(password)s" % self.admin_info],
ignore_failure=False)
except TestException:
suffix = str(int(suffix) + 1)
else:
break
return suffix
def get_cms_config(self):
if self._cms_config is None:
with io.open("%(CONFIG_PATH)s" % CONFIG, "rt") as f:
self._cms_config = json.load(f)
return self._cms_config
def admin_req(self, path, args=None, files=None):
browser = self.get_aws_browser()
return browser.do_request(self.AWS_BASE_URL + '/' + path, args, files)
def get_tasks(self):
"""Return the existing tasks
return ({string: {id: string, title: string}}): the tasks, as a
dictionary with the task name as key.
"""
r = self.admin_req('tasks')
groups = re.findall(r'''
<tr>\s*
<td><a\s+href="./task/(\d+)">(.*)</a></td>\s*
<td>(.*)</td>\s*
''', r.text, re.X)
tasks = {}
for g in groups:
id_, name, title = g
id_ = int(id_)
tasks[name] = {
'title': title,
'id': id_,
}
return tasks
def get_users(self, contest_id):
"""Return the existing users
return ({string: {id: string, firstname: string, lastname: string}):
the users, as a dictionary with the username as key.
"""
r = self.admin_req('contest/%s/users' % contest_id)
groups = re.findall(r'''
<tr> \s*
<td> \s* (.*) \s* </td> \s*
<td> \s* (.*) \s* </td> \s*
<td><a\s+href="./user/(\d+)">(.*)</a></td>
''', r.text, re.X)
users = {}
for g in groups:
firstname, lastname, id_, username = g
id_ = int(id_)
users[username] = {
'firstname': firstname,
'lastname': lastname,
'id': id_,
}
return users
def add_contest(self, **kwargs):
add_args = {
"name": kwargs.get('name'),
"description": kwargs.get('description'),
}
resp = self.admin_req('contests/add', args=add_args)
# Contest ID is returned as HTTP response.
page = resp.text
match = re.search(
r'<form enctype="multipart/form-data" '
r'action="../contest/([0-9]+)" '
r'method="POST" name="edit_contest" style="display:inline;">',
page)
if match is not None:
contest_id = int(match.groups()[0])
self.admin_req('contest/%s' % contest_id, args=kwargs)
return contest_id
else:
raise TestException("Unable to create contest.")
def add_task(self, **kwargs):
add_args = {
"name": kwargs.get('name'),
"title": kwargs.get('title'),
}
r = self.admin_req('tasks/add', args=add_args)
response = r.text
match_task_id = re.search(r'/task/([0-9]+)$', r.url)
match_dataset_id = re.search(r'/dataset/([0-9]+)', response)
if match_task_id and match_dataset_id:
task_id = int(match_task_id.group(1))
dataset_id = int(match_dataset_id.group(1))
edit_args = {}
for k, v in iteritems(kwargs):
edit_args[k.replace("{{dataset_id}}", str(dataset_id))] = v
r = self.admin_req('task/%s' % task_id, args=edit_args)
self.created_tasks[task_id] = kwargs
else:
raise TestException("Unable to create task.")
r = self.admin_req('contest/' + kwargs["contest_id"] + '/tasks/add',
args={"task_id": str(task_id)})
g = re.search('<input type="radio" name="task_id" value="' +
str(task_id) + '"/>', r.text)
if g:
return task_id
else:
raise TestException("Unable to assign task to contest.")
def add_manager(self, task_id, manager):
args = {}
files = [
('manager', manager),
]
dataset_id = self.get_task_active_dataset_id(task_id)
self.admin_req('dataset/%s/managers/add' % dataset_id,
files=files, args=args)
def get_task_active_dataset_id(self, task_id):
resp = self.admin_req('task/%s' % task_id)
page = resp.text
match = re.search(
r'id="title_dataset_([0-9]+).* \(Live\)</',
page)
if match is None:
raise TestException("Unable to create contest.")
dataset_id = int(match.groups()[0])
return dataset_id
def add_testcase(self, task_id, num, input_file, output_file, public):
files = [
('input', input_file),
('output', output_file),
]
args = {}
args["codename"] = "%03d" % num
if public:
args['public'] = '1'
dataset_id = self.get_task_active_dataset_id(task_id)
self.admin_req('dataset/%s/testcases/add' % dataset_id,
files=files, args=args)
def add_user(self, **kwargs):
r = self.admin_req('users/add', args=kwargs)
g = re.search(r'/user/([0-9]+)$', r.url)
if g:
user_id = int(g.group(1))
self.created_users[user_id] = kwargs
else:
raise TestException("Unable to create user.")
kwargs["user_id"] = user_id
r = self.admin_req('contest/%s/users/add' % kwargs["contest_id"],
args=kwargs)
g = re.search('<input type="radio" name="user_id" value="' +
str(user_id) + '"/>', r.text)
if g:
return user_id
else:
raise TestException("Unable to create participation.")
def add_existing_task(self, task_id, **kwargs):
"""Inform the framework of an existing task"""
self.created_tasks[task_id] = kwargs
def add_existing_user(self, user_id, **kwargs):
"""Inform the framework of an existing user"""
self.created_users[user_id] = kwargs
def cws_submit(self, task_id, user_id,
submission_format, filenames, language):
task = (task_id, self.created_tasks[task_id]['name'])
browser = self.get_cws_browser(user_id)
sr = SubmitRequest(browser, task, base_url=self.CWS_BASE_URL,
submission_format=submission_format,
filenames=filenames, language=language)
sr.execute()
submission_id = sr.get_submission_id()
if submission_id is None:
raise TestException("Failed to submit solution.")
return submission_id
def cws_submit_user_test(self, task_id, user_id,
submission_format, filenames, language):
task = (task_id, self.created_tasks[task_id]['name'])
browser = self.get_cws_browser(user_id)
sr = SubmitUserTestRequest(
browser, task, base_url=self.CWS_BASE_URL,
submission_format=submission_format,
filenames=filenames, language=language)
sr.execute()
user_test_id = sr.get_user_test_id()
if user_test_id is None:
raise TestException("Failed to submit user test.")
return user_test_id
def get_evaluation_result(self, contest_id, submission_id, timeout=60):
browser = self.get_aws_browser()
sleep_interval = 0.1
while timeout > 0:
timeout -= sleep_interval
sr = AWSSubmissionViewRequest(browser,
submission_id,
base_url=self.AWS_BASE_URL)
sr.execute()
result = sr.get_submission_info()
status = result['status']
if self.COMPLETED_STATUSES.search(status):
return result
if self.WAITING_STATUSES.search(status):
time.sleep(sleep_interval)
continue
raise TestException("Unknown submission status: %s" % status)
raise TestException("Waited too long for submission result.")
def get_user_test_result(self, contest_id, user_test_id, timeout=60):
browser = self.get_aws_browser()
sleep_interval = 0.1
while timeout > 0:
timeout -= sleep_interval
sr = AWSUserTestViewRequest(browser,
user_test_id,
base_url=self.AWS_BASE_URL)
sr.execute()
result = sr.get_user_test_info()
status = result['status']
if self.COMPLETED_STATUSES_USER_TEST.search(status):
return result
if self.WAITING_STATUSES_USER_TEST.search(status):
time.sleep(sleep_interval)
continue
raise TestException("Unknown user test status: %s" % status)
raise TestException("Waited too long for user test result.")
class Actor(threading.Thread):
"""Class that simulates the behaviour of a user of the system. It
performs some requests at randomized times (checking CMS pages,
doing submissions, ...), checking for their success or failure.
The probability that the users doing actions depends on the value
specified in an object called "metrics".
"""
def __init__(self, username, password, metrics, tasks,
log=None, base_url=None, submissions_path=None):
threading.Thread.__init__(self)
self.username = username
self.password = password
self.metrics = metrics
self.tasks = tasks
self.log = log
self.base_url = base_url
self.submissions_path = submissions_path
self.name = "Actor thread for user %s" % (self.username)
self.browser = Browser()
self.die = False
def run(self):
try:
print("Starting actor for user %s" % (self.username),
file=sys.stderr)
self.act()
except ActorDying:
print("Actor dying for user %s" % (self.username), file=sys.stderr)
def act(self):
"""Define the behaviour of the actor. Subclasses are expected
to overwrite this stub method properly.
"""
raise Exception("Not implemented. Please subclass Action"
"and overwrite act().")
def do_step(self, request):
self.wait_next()
self.log.total += 1
try:
request.execute()
except Exception as exc:
print("Unhandled exception while executing the request: %s" % exc,
file=sys.stderr)
return
self.log.__dict__[request.outcome] += 1
self.log.total_time += request.duration
self.log.max_time = max(self.log.max_time, request.duration)
self.log.store_to_file(request)
def wait_next(self):
"""Wait some time. At the moment it waits c*X seconds, where c
is the time_coeff parameter in metrics and X is an
exponentially distributed random variable, with parameter
time_lambda in metrics.
The total waiting time is divided in lots of little sleep()
call each one of 0.1 seconds, so that the waiting gets
interrupted if a die signal arrives.
If a die signal is received, an ActorDying exception is
raised.
"""
SLEEP_PERIOD = 0.1
time_to_wait = self.metrics['time_coeff'] * \
random.expovariate(self.metrics['time_lambda'])
sleep_num = time_to_wait // SLEEP_PERIOD
remaining_sleep = time_to_wait - (sleep_num * SLEEP_PERIOD)
for _ in range(sleep_num):
time.sleep(SLEEP_PERIOD)
if self.die:
raise ActorDying()
time.sleep(remaining_sleep)
if self.die:
raise ActorDying()
def login(self):
"""Log in and check to be logged in."""
self.do_step(HomepageRequest(self.browser,
self.username,
loggedin=False,
base_url=self.base_url))
lr = CWSLoginRequest(self.browser,
self.username,
self.password,
base_url=self.base_url)
self.browser.read_xsrf_token(lr.base_url)
self.do_step(lr)
self.do_step(HomepageRequest(self.browser,
self.username,
loggedin=True,
base_url=self.base_url))