def test_add_submission_too_early_submission(base_db): # check that we raise an error when the elapsed time was not large enough # between the new submission and the previous submission session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) # check that we have an awaiting time for the event event = (session.query(Event).filter( Event.name == event_name).one_or_none()) assert event.min_duration_between_submissions == 900 # make 2 submissions which are too close from each other for submission_idx, submission_name in enumerate( ['random_forest_10_10', 'too_early_submission']): path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) if submission_idx == 1: err_msg = 'You need to wait' with pytest.raises(TooEarlySubmissionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) else: add_submission(session, event_name, username, submission_name, path_submission)
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # check that the submissions have been copied for sub_id, _, _ in all_submissions: sub = get_submission_by_id(session, sub_id) assert os.path.exists(sub.path) assert os.path.exists(os.path.join(sub.path, 'classifier.py')) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 # check that the number of submissions for an event was updated event = session.query(Event).filter(Event.name == event_name).one_or_none() assert event.n_submissions == 1 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = read_config(ramp_config_template()) event_name, username = _setup_sign_up(session, config) ramp_config = generate_ramp_config(config) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def test_make_submission_resubmission(base_db): # check that resubmitting the a submission with the same name will raise # an error session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) # submitting the starting_kit which is used as the default submission for # the sandbox should raise an error err_msg = ('Submission "starting_kit" of team "test_user" at event ' '"iris_test" exists already') with pytest.raises(DuplicateSubmissionError, match=err_msg): add_submission(session, event_name, username, os.path.basename(ramp_config['ramp_sandbox_dir']), ramp_config['ramp_sandbox_dir']) # submitting twice a normal submission should raise an error as well submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) # first submission add_submission( session, event_name, username, submission_name, path_submission, ) # mock that we scored the submission set_submission_state(session, 5, 'scored') # second submission err_msg = ('Submission "random_forest_10_10" of team "test_user" at event ' '"iris_test" exists already') with pytest.raises(DuplicateSubmissionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # a resubmission can take place if it is tagged as "new" or failed # mock that the submission failed during the training set_submission_state(session, 5, 'training_error') add_submission(session, event_name, username, submission_name, path_submission) # mock that the submissions are new submissions set_submission_state(session, 5, 'new') add_submission(session, event_name, username, submission_name, path_submission)
def test_add_submission_wrong_submission_files(base_db): # check that we raise an error if the file required by the workflow is not # present in the submission or that it has the wrong extension session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submission_name = 'corrupted_submission' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) os.makedirs(path_submission) # case that there is not files in the submission err_msg = 'No file corresponding to the workflow element' with pytest.raises(MissingSubmissionFileError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # case that there is not file corresponding to the workflow component filename = os.path.join(path_submission, 'unknown_file.xxx') open(filename, "w+").close() err_msg = 'No file corresponding to the workflow element' with pytest.raises(MissingSubmissionFileError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # case that we have the correct filename but not the right extension filename = os.path.join(path_submission, 'classifier.xxx') open(filename, "w+").close() err_msg = 'All extensions "xxx" are unknown for the submission' with pytest.raises(MissingExtensionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission)
def test_is_accessible_code(session_toy_db): # create a third user add_user( session_toy_db, name='test_user_3', password='******', lastname='Test_3', firstname='User_3', email='*****@*****.**', access_level='user') approve_user(session_toy_db, 'test_user_3') event_name = 'iris_test' sign_up_team(session_toy_db, event_name, 'test_user_3') # simulate a user which is not authenticated user = get_user_by_name(session_toy_db, 'test_user_2') user.is_authenticated = False assert not is_accessible_code(session_toy_db, event_name, user.name) # simulate a user which authenticated and author of the submission to a # public event user.is_authenticated = True assert is_accessible_code(session_toy_db, event_name, user.name) # simulate an admin user user = get_user_by_name(session_toy_db, 'test_iris_admin') user.is_authenticated = True assert is_accessible_code(session_toy_db, event_name, 'test_iris_admin') # simulate a user which is not signed up to the event user = add_user(session_toy_db, 'xx', 'xx', 'xx', 'xx', 'xx', 'user') user.is_authenticated = True assert not is_accessible_code(session_toy_db, event_name, user.name) # simulate that the event is not publicly opened event = get_event(session_toy_db, event_name) past_public_opening = event.public_opening_timestamp tomorrow = datetime.datetime.utcnow() + datetime.timedelta(days=1) event.public_opening_timestamp = tomorrow session_toy_db.commit() assert is_accessible_code(session_toy_db, event_name, 'test_user_3') # Make a submission submission_name = 'random_forest_10_10' ramp_config = generate_ramp_config(read_config(ramp_config_template())) path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name ) sub = add_submission( session_toy_db, event_name, 'test_user_3', submission_name, path_submission ) # check that the user submitting the submission could access it assert is_accessible_code( session_toy_db, event_name, 'test_user_3', sub.id ) # change the admin of the team from ramp_database.model import Team, User team = (session_toy_db.query(Team) .filter(Team.name == 'test_user_3') .first()) user = (session_toy_db.query(User) .filter(User.name == 'test_user_2') .first()) team.admin_id = user.id team.admin = user session_toy_db.commit() # check that the admin can access the submission assert is_accessible_code( session_toy_db, event_name, 'test_user_2', sub.id ) # but others cannot assert not is_accessible_code( session_toy_db, event_name, 'test_user_3', sub.id ) event.public_opening_timestamp = past_public_opening session_toy_db.commit()
def sandbox(event_name): """Landing page for the user's sandbox. Parameters ---------- event_name : str The event name. """ event = get_event(db.session, event_name) if not is_accessible_event(db.session, event_name, flask_login.current_user.name): return redirect_to_user(u'{}: no event named "{}"'.format( flask_login.current_user.firstname, event_name)) if not is_accessible_code(db.session, event_name, flask_login.current_user.name): error_str = ('No access to sandbox for event {}. If you have ' 'already signed up, please wait for approval.'.format( event.name)) return redirect_to_user(error_str) # setup the webpage when loading # we use the code store in the sandbox to show to the user sandbox_submission = get_submission_by_name(db.session, event_name, flask_login.current_user.name, event.ramp_sandbox_name) event_team = get_event_team_by_name(db.session, event_name, flask_login.current_user.name) # initialize the form for the code # The amount of python magic we have to do for rendering a variable # number of textareas, named and populated at run time, is mind # boggling. # First we need to make sure CodeForm is empty # for name_code in CodeForm.names_codes: # name, _ = name_code # delattr(CodeForm, name) CodeForm.names_codes = [] # Then we create named fields in the CodeForm class for each editable # submission file. They have to be populated when the code_form object # is created, so we also create a code_form_kwargs dictionary and # populate it with the codes. code_form_kwargs = {} for submission_file in sandbox_submission.files: if submission_file.is_editable: f_field = submission_file.name setattr(CodeForm, f_field, StringField(u'Text', widget=TextArea())) code_form_kwargs[f_field] = submission_file.get_code() code_form_kwargs['prefix'] = 'code' code_form = CodeForm(**code_form_kwargs) # Then, to be able to iterate over the files in the sandbox.html # template, we also fill a separate table of pairs (file name, code). # The text areas in the template will then have to be created manually. for submission_file in sandbox_submission.files: if submission_file.is_editable: code_form.names_codes.append( (submission_file.name, submission_file.get_code())) # initialize the submission field and the the uploading form submit_form = SubmitForm(submission_name=event_team.last_submission_name, prefix='submit') upload_form = UploadForm(prefix='upload') admin = is_admin(db.session, event_name, flask_login.current_user.name) if request.method == 'GET': return render_template('sandbox.html', submission_names=sandbox_submission.f_names, code_form=code_form, submit_form=submit_form, upload_form=upload_form, event=event, admin=admin) if request.method == 'POST': if ('code-csrf_token' in request.form and code_form.validate_on_submit()): try: for submission_file in sandbox_submission.files: if submission_file.is_editable: old_code = submission_file.get_code() submission_file.set_code( request.form[submission_file.name]) new_code = submission_file.get_code() diff = '\n'.join( difflib.unified_diff(old_code.splitlines(), new_code.splitlines())) similarity = difflib.SequenceMatcher( a=old_code, b=new_code).ratio() if app.config['TRACK_USER_INTERACTION']: add_user_interaction( db.session, interaction='save', user=flask_login.current_user, event=event, submission_file=submission_file, diff=diff, similarity=similarity) except Exception as e: return redirect_to_sandbox(event, u'Error: {}'.format(e)) return redirect_to_sandbox( event, 'You submission has been saved. You can safely comeback to ' 'your sandbox later.', is_error=False, category='File saved') elif request.files: upload_f_name = secure_filename(request.files['file'].filename) upload_name = upload_f_name.split('.')[0] # TODO: create a get_function upload_workflow_element = WorkflowElement.query.filter_by( name=upload_name, workflow=event.workflow).one_or_none() if upload_workflow_element is None: return redirect_to_sandbox( event, u'{} is not in the file list.'.format(upload_f_name)) # TODO: create a get_function submission_file = SubmissionFile.query.filter_by( submission=sandbox_submission, workflow_element=upload_workflow_element).one() if submission_file.is_editable: old_code = submission_file.get_code() tmp_f_name = os.path.join(tempfile.gettempdir(), upload_f_name) request.files['file'].save(tmp_f_name) file_length = os.stat(tmp_f_name).st_size if (upload_workflow_element.max_size is not None and file_length > upload_workflow_element.max_size): return redirect_to_sandbox( event, u'File is too big: {} exceeds max size {}'.format( file_length, upload_workflow_element.max_size)) if submission_file.is_editable: try: with open(tmp_f_name) as f: code = f.read() submission_file.set_code(code) except Exception as e: return redirect_to_sandbox(event, u'Error: {}'.format(e)) else: # non-editable files are not verified for now dst = os.path.join(sandbox_submission.path, upload_f_name) shutil.copy2(tmp_f_name, dst) logger.info(u'{} uploaded {} in {}'.format( flask_login.current_user.name, upload_f_name, event)) if submission_file.is_editable: new_code = submission_file.get_code() diff = '\n'.join( difflib.unified_diff(old_code.splitlines(), new_code.splitlines())) similarity = difflib.SequenceMatcher(a=old_code, b=new_code).ratio() if app.config['TRACK_USER_INTERACTION']: add_user_interaction(db.session, interaction='upload', user=flask_login.current_user, event=event, submission_file=submission_file, diff=diff, similarity=similarity) else: if app.config['TRACK_USER_INTERACTION']: add_user_interaction(db.session, interaction='upload', user=flask_login.current_user, event=event, submission_file=submission_file) return redirect(request.referrer) # TODO: handle different extensions for the same workflow element # ie: now we let upload eg external_data.bla, and only fail at # submission, without giving a message elif ('submit-csrf_token' in request.form and submit_form.validate_on_submit()): new_submission_name = request.form['submit-submission_name'] if not 4 < len(new_submission_name) < 20: return redirect_to_sandbox( event, 'Submission name should have length between 4 and ' '20 characters.') try: new_submission_name.encode('ascii') except Exception as e: return redirect_to_sandbox(event, u'Error: {}'.format(e)) try: new_submission = add_submission(db.session, event_name, event_team.team.name, new_submission_name, sandbox_submission.path) except DuplicateSubmissionError: return redirect_to_sandbox( event, u'Submission {} already exists. Please change the name.'. format(new_submission_name)) except MissingExtensionError as e: return redirect_to_sandbox(event, 'Missing extension') except TooEarlySubmissionError as e: return redirect_to_sandbox(event, str(e)) logger.info(u'{} submitted {} for {}.'.format( flask_login.current_user.name, new_submission.name, event_team)) if event.is_send_submitted_mails: admin_users = User.query.filter_by(access_level='admin') for admin in admin_users: subject = 'Submission {} sent for training'.format( new_submission.name) body = """A new submission have been submitted: event: {} user: {} submission: {} submission path: {} """.format(event_team.event.name, flask_login.current_user.name, new_submission.name, new_submission.path) send_mail(admin, subject, body) if app.config['TRACK_USER_INTERACTION']: add_user_interaction(db.session, interaction='submit', user=flask_login.current_user, event=event, submission=new_submission) return redirect_to_sandbox(event, u'{} submitted {} for {}'.format( flask_login.current_user.firstname, new_submission.name, event_team), is_error=False, category='Submission') admin = is_admin(db.session, event_name, flask_login.current_user.name) return render_template('sandbox.html', submission_names=sandbox_submission.f_names, code_form=code_form, submit_form=submit_form, upload_form=upload_form, event=event, admin=admin)