def test_add_submission_too_early_submission(base_db):
# check that we raise an error when the elapsed time was not large enough
# between the new submission and the previous submission
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
# check that we have an awaiting time for the event
event = (session.query(Event).filter(
Event.name == event_name).one_or_none())
assert event.min_duration_between_submissions == 900
# make 2 submissions which are too close from each other
for submission_idx, submission_name in enumerate(
['random_forest_10_10', 'too_early_submission']):
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
if submission_idx == 1:
err_msg = 'You need to wait'
with pytest.raises(TooEarlySubmissionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
else:
add_submission(session, event_name, username, submission_name,
path_submission)
def test_add_submission_create_new_submission(base_db):
# check that we can make a new submission to the database
# it will require to have already a team and an event
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
add_submission(session, event_name, username, submission_name,
path_submission)
all_submissions = get_submissions(session, event_name, None)
# check that the submissions have been copied
for sub_id, _, _ in all_submissions:
sub = get_submission_by_id(session, sub_id)
assert os.path.exists(sub.path)
assert os.path.exists(os.path.join(sub.path, 'classifier.py'))
# `sign_up_team` make a submission (sandbox) by user. This submission will
# be the third submission.
assert len(all_submissions) == 3
# check that the number of submissions for an event was updated
event = session.query(Event).filter(Event.name == event_name).one_or_none()
assert event.n_submissions == 1
submission = get_submission_by_name(session, event_name, username,
submission_name)
assert submission.name == submission_name
submission_file = submission.files[0]
assert submission_file.name == 'classifier'
assert submission_file.extension == 'py'
assert (os.path.join('submission_000000005', 'classifier.py')
in submission_file.path)
def test_add_submission_create_new_submission(base_db):
# check that we can make a new submission to the database
# it will require to have already a team and an event
session = base_db
config = read_config(ramp_config_template())
event_name, username = _setup_sign_up(session, config)
ramp_config = generate_ramp_config(config)
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
add_submission(session, event_name, username, submission_name,
path_submission)
all_submissions = get_submissions(session, event_name, None)
# `sign_up_team` make a submission (sandbox) by user. This submission will
# be the third submission.
assert len(all_submissions) == 3
submission = get_submission_by_name(session, event_name, username,
submission_name)
assert submission.name == submission_name
submission_file = submission.files[0]
assert submission_file.name == 'classifier'
assert submission_file.extension == 'py'
assert (os.path.join('submission_000000005', 'classifier.py')
in submission_file.path)
def test_make_submission_resubmission(base_db):
# check that resubmitting the a submission with the same name will raise
# an error
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
# submitting the starting_kit which is used as the default submission for
# the sandbox should raise an error
err_msg = ('Submission "starting_kit" of team "test_user" at event '
'"iris_test" exists already')
with pytest.raises(DuplicateSubmissionError, match=err_msg):
add_submission(session, event_name, username,
os.path.basename(ramp_config['ramp_sandbox_dir']),
ramp_config['ramp_sandbox_dir'])
# submitting twice a normal submission should raise an error as well
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
# first submission
add_submission(
session,
event_name,
username,
submission_name,
path_submission,
)
# mock that we scored the submission
set_submission_state(session, 5, 'scored')
# second submission
err_msg = ('Submission "random_forest_10_10" of team "test_user" at event '
'"iris_test" exists already')
with pytest.raises(DuplicateSubmissionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# a resubmission can take place if it is tagged as "new" or failed
# mock that the submission failed during the training
set_submission_state(session, 5, 'training_error')
add_submission(session, event_name, username, submission_name,
path_submission)
# mock that the submissions are new submissions
set_submission_state(session, 5, 'new')
add_submission(session, event_name, username, submission_name,
path_submission)
def test_add_submission_wrong_submission_files(base_db):
# check that we raise an error if the file required by the workflow is not
# present in the submission or that it has the wrong extension
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submission_name = 'corrupted_submission'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
os.makedirs(path_submission)
# case that there is not files in the submission
err_msg = 'No file corresponding to the workflow element'
with pytest.raises(MissingSubmissionFileError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# case that there is not file corresponding to the workflow component
filename = os.path.join(path_submission, 'unknown_file.xxx')
open(filename, "w+").close()
err_msg = 'No file corresponding to the workflow element'
with pytest.raises(MissingSubmissionFileError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# case that we have the correct filename but not the right extension
filename = os.path.join(path_submission, 'classifier.xxx')
open(filename, "w+").close()
err_msg = 'All extensions "xxx" are unknown for the submission'
with pytest.raises(MissingExtensionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
def test_is_accessible_code(session_toy_db):
# create a third user
add_user(
session_toy_db, name='test_user_3', password='******',
lastname='Test_3', firstname='User_3',
email='*****@*****.**', access_level='user')
approve_user(session_toy_db, 'test_user_3')
event_name = 'iris_test'
sign_up_team(session_toy_db, event_name, 'test_user_3')
# simulate a user which is not authenticated
user = get_user_by_name(session_toy_db, 'test_user_2')
user.is_authenticated = False
assert not is_accessible_code(session_toy_db, event_name, user.name)
# simulate a user which authenticated and author of the submission to a
# public event
user.is_authenticated = True
assert is_accessible_code(session_toy_db, event_name, user.name)
# simulate an admin user
user = get_user_by_name(session_toy_db, 'test_iris_admin')
user.is_authenticated = True
assert is_accessible_code(session_toy_db, event_name, 'test_iris_admin')
# simulate a user which is not signed up to the event
user = add_user(session_toy_db, 'xx', 'xx', 'xx', 'xx', 'xx', 'user')
user.is_authenticated = True
assert not is_accessible_code(session_toy_db, event_name, user.name)
# simulate that the event is not publicly opened
event = get_event(session_toy_db, event_name)
past_public_opening = event.public_opening_timestamp
tomorrow = datetime.datetime.utcnow() + datetime.timedelta(days=1)
event.public_opening_timestamp = tomorrow
session_toy_db.commit()
assert is_accessible_code(session_toy_db, event_name, 'test_user_3')
# Make a submission
submission_name = 'random_forest_10_10'
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name
)
sub = add_submission(
session_toy_db, event_name, 'test_user_3', submission_name,
path_submission
)
# check that the user submitting the submission could access it
assert is_accessible_code(
session_toy_db, event_name, 'test_user_3', sub.id
)
# change the admin of the team
from ramp_database.model import Team, User
team = (session_toy_db.query(Team)
.filter(Team.name == 'test_user_3')
.first())
user = (session_toy_db.query(User)
.filter(User.name == 'test_user_2')
.first())
team.admin_id = user.id
team.admin = user
session_toy_db.commit()
# check that the admin can access the submission
assert is_accessible_code(
session_toy_db, event_name, 'test_user_2', sub.id
)
# but others cannot
assert not is_accessible_code(
session_toy_db, event_name, 'test_user_3', sub.id
)
event.public_opening_timestamp = past_public_opening
session_toy_db.commit()
def sandbox(event_name):
"""Landing page for the user's sandbox.
Parameters
----------
event_name : str
The event name.
"""
event = get_event(db.session, event_name)
if not is_accessible_event(db.session, event_name,
flask_login.current_user.name):
return redirect_to_user(u'{}: no event named "{}"'.format(
flask_login.current_user.firstname, event_name))
if not is_accessible_code(db.session, event_name,
flask_login.current_user.name):
error_str = ('No access to sandbox for event {}. If you have '
'already signed up, please wait for approval.'.format(
event.name))
return redirect_to_user(error_str)
# setup the webpage when loading
# we use the code store in the sandbox to show to the user
sandbox_submission = get_submission_by_name(db.session, event_name,
flask_login.current_user.name,
event.ramp_sandbox_name)
event_team = get_event_team_by_name(db.session, event_name,
flask_login.current_user.name)
# initialize the form for the code
# The amount of python magic we have to do for rendering a variable
# number of textareas, named and populated at run time, is mind
# boggling.
# First we need to make sure CodeForm is empty
# for name_code in CodeForm.names_codes:
# name, _ = name_code
# delattr(CodeForm, name)
CodeForm.names_codes = []
# Then we create named fields in the CodeForm class for each editable
# submission file. They have to be populated when the code_form object
# is created, so we also create a code_form_kwargs dictionary and
# populate it with the codes.
code_form_kwargs = {}
for submission_file in sandbox_submission.files:
if submission_file.is_editable:
f_field = submission_file.name
setattr(CodeForm, f_field, StringField(u'Text', widget=TextArea()))
code_form_kwargs[f_field] = submission_file.get_code()
code_form_kwargs['prefix'] = 'code'
code_form = CodeForm(**code_form_kwargs)
# Then, to be able to iterate over the files in the sandbox.html
# template, we also fill a separate table of pairs (file name, code).
# The text areas in the template will then have to be created manually.
for submission_file in sandbox_submission.files:
if submission_file.is_editable:
code_form.names_codes.append(
(submission_file.name, submission_file.get_code()))
# initialize the submission field and the the uploading form
submit_form = SubmitForm(submission_name=event_team.last_submission_name,
prefix='submit')
upload_form = UploadForm(prefix='upload')
admin = is_admin(db.session, event_name, flask_login.current_user.name)
if request.method == 'GET':
return render_template('sandbox.html',
submission_names=sandbox_submission.f_names,
code_form=code_form,
submit_form=submit_form,
upload_form=upload_form,
event=event,
admin=admin)
if request.method == 'POST':
if ('code-csrf_token' in request.form
and code_form.validate_on_submit()):
try:
for submission_file in sandbox_submission.files:
if submission_file.is_editable:
old_code = submission_file.get_code()
submission_file.set_code(
request.form[submission_file.name])
new_code = submission_file.get_code()
diff = '\n'.join(
difflib.unified_diff(old_code.splitlines(),
new_code.splitlines()))
similarity = difflib.SequenceMatcher(
a=old_code, b=new_code).ratio()
if app.config['TRACK_USER_INTERACTION']:
add_user_interaction(
db.session,
interaction='save',
user=flask_login.current_user,
event=event,
submission_file=submission_file,
diff=diff,
similarity=similarity)
except Exception as e:
return redirect_to_sandbox(event, u'Error: {}'.format(e))
return redirect_to_sandbox(
event,
'You submission has been saved. You can safely comeback to '
'your sandbox later.',
is_error=False,
category='File saved')
elif request.files:
upload_f_name = secure_filename(request.files['file'].filename)
upload_name = upload_f_name.split('.')[0]
# TODO: create a get_function
upload_workflow_element = WorkflowElement.query.filter_by(
name=upload_name, workflow=event.workflow).one_or_none()
if upload_workflow_element is None:
return redirect_to_sandbox(
event,
u'{} is not in the file list.'.format(upload_f_name))
# TODO: create a get_function
submission_file = SubmissionFile.query.filter_by(
submission=sandbox_submission,
workflow_element=upload_workflow_element).one()
if submission_file.is_editable:
old_code = submission_file.get_code()
tmp_f_name = os.path.join(tempfile.gettempdir(), upload_f_name)
request.files['file'].save(tmp_f_name)
file_length = os.stat(tmp_f_name).st_size
if (upload_workflow_element.max_size is not None
and file_length > upload_workflow_element.max_size):
return redirect_to_sandbox(
event, u'File is too big: {} exceeds max size {}'.format(
file_length, upload_workflow_element.max_size))
if submission_file.is_editable:
try:
with open(tmp_f_name) as f:
code = f.read()
submission_file.set_code(code)
except Exception as e:
return redirect_to_sandbox(event, u'Error: {}'.format(e))
else:
# non-editable files are not verified for now
dst = os.path.join(sandbox_submission.path, upload_f_name)
shutil.copy2(tmp_f_name, dst)
logger.info(u'{} uploaded {} in {}'.format(
flask_login.current_user.name, upload_f_name, event))
if submission_file.is_editable:
new_code = submission_file.get_code()
diff = '\n'.join(
difflib.unified_diff(old_code.splitlines(),
new_code.splitlines()))
similarity = difflib.SequenceMatcher(a=old_code,
b=new_code).ratio()
if app.config['TRACK_USER_INTERACTION']:
add_user_interaction(db.session,
interaction='upload',
user=flask_login.current_user,
event=event,
submission_file=submission_file,
diff=diff,
similarity=similarity)
else:
if app.config['TRACK_USER_INTERACTION']:
add_user_interaction(db.session,
interaction='upload',
user=flask_login.current_user,
event=event,
submission_file=submission_file)
return redirect(request.referrer)
# TODO: handle different extensions for the same workflow element
# ie: now we let upload eg external_data.bla, and only fail at
# submission, without giving a message
elif ('submit-csrf_token' in request.form
and submit_form.validate_on_submit()):
new_submission_name = request.form['submit-submission_name']
if not 4 < len(new_submission_name) < 20:
return redirect_to_sandbox(
event, 'Submission name should have length between 4 and '
'20 characters.')
try:
new_submission_name.encode('ascii')
except Exception as e:
return redirect_to_sandbox(event, u'Error: {}'.format(e))
try:
new_submission = add_submission(db.session, event_name,
event_team.team.name,
new_submission_name,
sandbox_submission.path)
except DuplicateSubmissionError:
return redirect_to_sandbox(
event,
u'Submission {} already exists. Please change the name.'.
format(new_submission_name))
except MissingExtensionError as e:
return redirect_to_sandbox(event, 'Missing extension')
except TooEarlySubmissionError as e:
return redirect_to_sandbox(event, str(e))
logger.info(u'{} submitted {} for {}.'.format(
flask_login.current_user.name, new_submission.name,
event_team))
if event.is_send_submitted_mails:
admin_users = User.query.filter_by(access_level='admin')
for admin in admin_users:
subject = 'Submission {} sent for training'.format(
new_submission.name)
body = """A new submission have been submitted:
event: {}
user: {}
submission: {}
submission path: {}
""".format(event_team.event.name,
flask_login.current_user.name,
new_submission.name, new_submission.path)
send_mail(admin, subject, body)
if app.config['TRACK_USER_INTERACTION']:
add_user_interaction(db.session,
interaction='submit',
user=flask_login.current_user,
event=event,
submission=new_submission)
return redirect_to_sandbox(event,
u'{} submitted {} for {}'.format(
flask_login.current_user.firstname,
new_submission.name, event_team),
is_error=False,
category='Submission')
admin = is_admin(db.session, event_name, flask_login.current_user.name)
return render_template('sandbox.html',
submission_names=sandbox_submission.f_names,
code_form=code_form,
submit_form=submit_form,
upload_form=upload_form,
event=event,
admin=admin)
