def delete_solver(database, solver_id):
db = models.get_database(database) or abort(404)
solver = db.session.query(db.Solver).get(solver_id) or abort(404)
if solver.user.idUser != g.User.idUser: abort(401)
try:
for solver_binary in solver.binaries:
for solver_config in solver_binary.solver_configurations:
for pi in solver_config.parameter_instances:
db.session.delete(pi)
db.session.commit()
db.session.delete(solver_config)
db.session.commit()
db.session.delete(solver_binary)
for p in solver.parameters:
db.session.delete(p)
db.session.delete(solver)
db.session.commit()
flash('Solver deleted successfully.')
except Exception as e:
print e
db.session.rollback()
flash('Could not delete solver. Please contact an administrator.')
return redirect(
url_for('accounts.list_solvers', database=database, user_id=None))
def decorated_f(*args, **kwargs):
db = models.get_database(kwargs['database']) or abort(404)
if session.get('logged_in'):
g.User = db.session.query(db.User).get(session['idUser'])
if g.User.admin: session['admin'] = True
else:
g.User = None
if db.is_competition() and db.competition_phase() == 7:
session.pop('logged_in', None)
session.pop('idUser', None)
g.User = None
# if db.is_competition() and db.competition_phase() == 5:
# def redirect_f(*args, **kwargs):
# return redirect(url_for('frontend.experiments_index',
# database=kwargs['database']))
# if not g.User or not g.User.admin:
# session.pop('logged_in', None)
# flash('Website offline for competition computations.')
# return redirect_f(*args, **kwargs)
if db.is_competition() and db.competition_phase() < 7:
def redirect_f(*args, **kwargs):
return redirect(url_for('accounts.login',
database=kwargs['database']))
if not g.User or not session.get('logged_in') or \
session.get('idUser', None) is None:
return redirect_f(*args, **kwargs)
if session.get('database') != kwargs['database']:
return redirect_f(*args, **kwargs)
return f(*args, **kwargs)
def change_password(database, reset_hash):
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).filter_by(
activation_hash='pw_reset_' + reset_hash).first() or abort(404)
form = forms.ChangePasswordForm()
if form.validate_on_submit():
user.activation_hash = ''
user.password = password_hash(form.password.data)
try:
db.session.commit()
flash('Password changed.')
except Exception as e:
print e
flash(
'Could not set password in db. Please contact an administrator.'
)
return redirect(
url_for('frontend.experiments_index', database=database))
return render('/accounts/change_password.html',
db=db,
database=database,
form=form,
reset_hash=reset_hash)
def list_solver_descriptions(database):
db = models.get_database(database) or abort(404)
solvers = [s for s in db.session.query(db.Solver).all() if s.User_idUser]
filtered_solvers = []
for s in solvers:
if not any(
ss.description_pdf == s.description_pdf and s.description_pdf
for ss in filtered_solvers):
filtered_solvers.append(s)
solvers_by_category = dict()
filtered_solvers_by_category = dict()
for category in db.session.query(db.CompetitionCategory).all():
solvers_by_category[category] = [
s for s in category.solvers if s.User_idUser
]
filtered_solvers_by_category[category] = []
for s in solvers_by_category[category]:
if not any(ss.description_pdf == s.description_pdf
and s.description_pdf
for ss in filtered_solvers_by_category[category]):
filtered_solvers_by_category[category].append(s)
return render('/accounts/list_solver_descriptions.html',
database=database,
categories=sorted(solvers_by_category.keys(),
key=lambda c: c.name),
db=db,
solvers=solvers,
solvers_by_category=solvers_by_category,
sorted=sorted,
filtered_solvers=filtered_solvers,
filtered_solvers_by_category=filtered_solvers_by_category)
def activate(database, activation_hash):
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).filter_by(activation_hash=activation_hash).first() or abort(404)
user.activation_hash = ""
try:
db.session.commit()
user = db.session.query(db.User).filter_by(email=user.email).first()
msg = Message("[" + db.label + "][Admin] Account was activated",
recipients=[config.DEFAULT_MAIL_SENDER])
msg.body = ("The following account was just activated by a user:\n\n" + \
"Last name: %s\n" \
"First name: %s\n" \
"E-mail: %s\n" \
"Postal address: %s\n" \
"Affiliation: %s\n" \
"Affiliation type: %s\n" \
"Country: %s\n\n\n" \
"Use the following link to verify this user: " + request.url_root[:-1] + url_for(
'accounts.verify_user', database=database, user_id=user.idUser)) \
% (
user.lastname, user.firstname, user.email, user.postal_address, user.affiliation, user.affiliation_type,
user.country)
mail.send(msg)
flash('Account activated. You will be able to log in when your account was verified by an administrator.')
except Exception as e:
db.session.rollback()
print e
flash('Could not activate account. Please contact an administrator.')
return redirect(url_for('frontend.experiments_index',
database=database))
def list_benchmarks(database, user_id=None):
""" Lists all benchmarks that the currently logged in user submitted.
"""
db = models.get_database(database) or abort(404)
uploaded_files = {}
directory = os.path.join(config.UPLOAD_FOLDER, database)
for file in os.listdir(directory):
if os.path.isdir(os.path.join(directory, file)):
for user_dir in os.listdir(os.path.join(directory, file)):
if (is_admin() and user_id and user_dir == str(user_id)) or (
not user_id and user_dir == str(g.User.idUser)):
if not file in uploaded_files:
uploaded_files[file] = list()
files = os.listdir(os.path.join(directory, file, user_dir))
files_full_path = [
os.path.join(directory, file, user_dir, f)
for f in files
]
uploaded_files[file] += zip(files, [
time.ctime(os.path.getmtime(f))
for f in files_full_path
])
return render('/accounts/list_benchmarks.html',
database=database,
db=db,
uploaded_files=uploaded_files,
user_id=user_id)
def activate(database, activation_hash):
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).filter_by(
activation_hash=activation_hash).first() or abort(404)
user.activation_hash = ""
try:
db.session.commit()
user = db.session.query(db.User).filter_by(email=user.email).first()
msg = Message("[" + db.label + "][Admin] Account was activated",
recipients=[config.DEFAULT_MAIL_SENDER])
msg.body = ("The following account was just activated by a user:\n\n" + \
"Last name: %s\n" \
"First name: %s\n" \
"E-mail: %s\n" \
"Postal address: %s\n" \
"Affiliation: %s\n" \
"Affiliation type: %s\n" \
"Country: %s\n\n\n" \
"Use the following link to verify this user: " + request.url_root[:-1] + url_for(
'accounts.verify_user', database=database, user_id=user.idUser)) \
% (
user.lastname, user.firstname, user.email, user.postal_address, user.affiliation, user.affiliation_type,
user.country)
mail.send(msg)
flash(
'Account activated. You will be able to log in when your account was verified by an administrator.'
)
except Exception as e:
db.session.rollback()
print e
flash('Could not activate account. Please contact an administrator.')
return redirect(url_for('frontend.experiments_index', database=database))
def verify_user(database, user_id):
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).get(user_id) or abort(404)
if user.verified:
flash('User already verified.')
return redirect(
url_for('frontend.experiments_index', database=database))
user.verified = True
try:
db.session.commit()
msg = Message("[" + db.label + "] Account verified",
recipients=[user.email])
msg.body = "Dear " + user.firstname + " " + user.lastname + ",\n\n" + \
"Your account was verified and you can now log in:\n" + \
request.url_root[:-1] + url_for('accounts.login', database=database)
mail.send(msg)
flash('Verified the account.')
except Exception as e:
db.session.rollback()
flash(
"Couldn't update verification status of user or send the notification mail: "
+ str(e))
return redirect(url_for('frontend.experiments_index', database=database))
def action(db_name=('db', 'EDACC'),
name=('name', ''),
file_path=('filename', '')):
"""Add a new property value type to the database.
Arguments in order:
"""
db = models.get_database(db_name)
p = db.session.query(db.PropertyValueType).filter_by(name=name).first()
if p is None:
p = db.PropertyValueType()
import os
try:
p.typeClass = file(file_path, 'rb').read()
except Exception as e:
print "Couldn't open file:", e
return
p.name = name
p.typeClassFileName = os.path.basename(file_path)
p.isDefault = True
db.session.add(p)
try:
db.session.commit()
print "Property value type saved."
except Exception as e:
print "Couldn't save property value type to database:", e
db.session.rollback()
def action(db_name=('db', 'EDACC')):
db = models.get_database(db_name)
instances = db.session.query(db.Instance).all()
for p in db.session.query(db.Property).filter_by(propertyType=0):
pat = re.compile(p.regExpression)
for instance in instances:
for pv in instance.properties:
if pv.property == p:
db.session.delete(pv)
db.session.commit()
break
if p.propertySource == 0:
text = instance.instance
elif p.propertySource == 1:
text = instance.name
m = re.search(pat, text)
if m is not None:
val = m.groups()[-1]
pv = db.InstanceProperties()
pv.property = p
pv.instance = instance
pv.value = val
db.session.add(pv)
try:
db.session.commit()
print "Instance property values calculated."
except Exception as e:
print "Error while saving instance property values", e
db.session.rollback()
def action(db_name=('db','EDACC'), propertyType=('type', 0),
propertySource=('source', 0), name=('name', ''), description=('desc', ''),
regExp=('regExp', ''), value_type=('value_type', ''),
multiple=('multiple', 0)):
"""Add a new property to the database.
propertyType: Instance = 0, Result = 1
propertySource: Instance = 0, InstanceName = 1, LauncherOutput = 2, Parameter = 3, SolverOutput = 4, VerifierOutput = 5, WatcherOutput = 6
value_type: name of a PropertyValueType
multiple: multiple occurences 1/0 for true/false
Arguments in order:
"""
db = models.get_database(db_name)
p = db.session.query(db.Property).filter_by(name=name).first()
if p is None:
p = db.Property()
p.propertyType = int(propertyType)
p.propertySource = int(propertySource)
p.name = name
p.description = description
p.regExpression = regExp
p.propertyValueType = value_type
p.isDefault = True
p.multipleOccourence = int(multiple)
db.session.add(p)
try:
db.session.commit()
print "Property saved."
except Exception as e:
print "Couldn't save property to database:", e
db.session.rollback()
def action(db_name=('db','EDACC'), name=('name', ''), file_path=('filename', '')):
"""Add a new property value type to the database.
Arguments in order:
"""
db = models.get_database(db_name)
p = db.session.query(db.PropertyValueType).filter_by(name=name).first()
if p is None:
p = db.PropertyValueType()
import os
try:
p.typeClass = file(file_path, 'rb').read()
except Exception as e:
print "Couldn't open file:", e
return
p.name = name
p.typeClassFileName = os.path.basename(file_path)
p.isDefault = True
db.session.add(p)
try:
db.session.commit()
print "Property value type saved."
except Exception as e:
print "Couldn't save property value type to database:", e
db.session.rollback()
def action(db_name=('db','EDACC')):
db = models.get_database(db_name)
instances = db.session.query(db.Instance).all()
for p in db.session.query(db.Property).filter_by(propertyType=0):
pat = re.compile(p.regExpression)
for instance in instances:
for pv in instance.properties:
if pv.property == p:
db.session.delete(pv)
db.session.commit()
break
if p.propertySource == 0:
text = instance.instance
elif p.propertySource == 1:
text = instance.name
m = re.search(pat, text)
if m is not None:
val = m.groups()[-1]
pv = db.InstanceProperties()
pv.property = p
pv.instance = instance
pv.value = val
db.session.add(pv)
try:
db.session.commit()
print "Instance property values calculated."
except Exception as e:
print "Error while saving instance property values", e
db.session.rollback()
def borg_explorer_data(database, experiment_id):
db = models.get_database(database) or abort(404)
experiment = db.session.query(db.Experiment).get(experiment_id) or abort(404)
type = request.args.get('type')
if type == 'categories.json':
return json_dumps([{"path": experiment.name, "name": experiment.name}])
last_modified_job = db.session.query(func.max(db.ExperimentResult.date_modified))\
.filter_by(experiment=experiment).first()
job_count = db.session.query(db.ExperimentResult).filter_by(experiment=experiment).count()
@synchronized
@cache.memoize(7*24*60*60)
def get_data(database, experiment_id, job_count, last_modified_job):
runs = db.session.query(db.ExperimentResult) \
.filter(db.ExperimentResult.Experiment_idExperiment==experiment_id) \
.filter(not_(db.ExperimentResult.status.in_(STATUS_PROCESSING))).order_by('idJob').all()
return CategoryData().fit([(0, r.instance.name, r.result_code.description, r.resultTime, 0, r.solver_configuration.name, 0) for r in runs])
data = get_data(database, experiment_id, job_count, last_modified_job)
if type == 'runs.json':
return json_dumps(data.table)
elif type == 'solvers.json':
return json_dumps(data.solvers)
elif type == 'instances.json':
return json_dumps(data.instances)
elif type == 'membership.json':
return json_dumps(data.model._tclass_res_LN.T.tolist())
elif type == 'projection.json':
return json_dumps(data.projection_N2.tolist())
else:
abort(404)
def get_experiment_results(database, experiment_id):
db = models.get_database(database) or abort(404)
exp = db.session.query(db.Experiment).get(experiment_id) or abort(404)
results = db.session.query(db.ExperimentResult).filter_by(experiment=exp).all()
return json_dumps({
"experiment_id": experiment_id,
"results": [j.to_json() for j in results]
})
def experiment_statistics(database, experiment_id):
db = models.get_database(database) or abort(404)
exp = db.session.query(db.Experiment).get(experiment_id) or abort(404)
total_time = db.session.query(func.sum(db.ExperimentResult.resultTime)).filter_by(experiment=exp).first()[0]
return json_dumps({
'total_time': total_time,
'total_time_hours': total_time / 60 / 60,
})
def list_submitted_solvers(database):
db = models.get_database(database) or abort(404)
solvers = db.session.query(db.Solver).filter(
db.Solver.User_idUser != None).order_by(db.Solver.name).all()
return render("/accounts/list_submitted_solvers.html",
database=database,
db=db,
solvers=solvers)
def borg_explorer(database, experiment_id):
db = models.get_database(database) or abort(404)
experiment = db.session.query(
db.Experiment).get(experiment_id) or abort(404)
return render('borgexplorer/index.html',
db=db,
database=database,
experiment=experiment)
def get_experiment_results(database, experiment_id):
db = models.get_database(database) or abort(404)
exp = db.session.query(db.Experiment).get(experiment_id) or abort(404)
results = db.session.query(
db.ExperimentResult).filter_by(experiment=exp).all()
return json_dumps({
"experiment_id": experiment_id,
"results": [j.to_json() for j in results]
})
def experiment_statistics(database, experiment_id):
db = models.get_database(database) or abort(404)
exp = db.session.query(db.Experiment).get(experiment_id) or abort(404)
total_time = db.session.query(func.sum(
db.ExperimentResult.resultTime)).filter_by(experiment=exp).first()[0]
return json_dumps({
'total_time': total_time,
'total_time_hours': total_time / 60 / 60,
})
def statistics(database):
db = models.get_database(database) or abort(404)
jobs_running = db.session.query(db.ExperimentResult).filter_by(status=0).count()
total_time = db.session.query(func.sum(db.ExperimentResult.resultTime)).first()[0]
return json_dumps({
'jobs_running': jobs_running,
'total_time': total_time,
'total_time_days': total_time / 60 / 60 / 24,
})
def statistics(database):
db = models.get_database(database) or abort(404)
jobs_running = db.session.query(
db.ExperimentResult).filter_by(status=0).count()
total_time = db.session.query(func.sum(
db.ExperimentResult.resultTime)).first()[0]
return json_dumps({
'jobs_running': jobs_running,
'total_time': total_time,
'total_time_days': total_time / 60 / 60 / 24,
})
def action(db_name=('db', 'EDACC'), experiment=('experiment', '')):
"""Calculate result properties for the jobs of the experiment with the name passed in."""
db = models.get_database(db_name)
exp = db.session.query(
db.Experiment).filter_by(name=experiment).first()
if exp is None:
print "Experiment doesn't exist"
return
props = db.session.query(db.Property).filter_by(propertyType=1)
for p in props:
pat = re.compile(p.regExpression)
for result in db.session.query(db.ExperimentResult).filter(
db.ExperimentResult.resultCode.like('1%')).filter_by(
experiment=exp).all():
for pv in result.properties:
if pv.property == p:
for pvv in pv.values:
db.session.delete(pvv)
db.session.commit()
db.session.delete(pv)
db.session.commit()
if p.propertySource == 2:
output = result.launcherOutput
elif p.propertySource == 4:
output = result.solverOutput
elif p.propertySource == 5:
output = result.verifierOutput
elif p.propertySource == 6:
output = result.watcherOutput
m = re.search(pat, output)
if m is not None:
val = m.groups()[-1]
pv = db.ExperimentResultProperty()
pv.property = p
pv.experiment_result = result
db.session.add(pv)
db.session.flush()
pvv = db.ResultPropertyValue()
pvv.value = val
pvv.order = 0
pvv.experiment_result_property = pv
db.session.add(pvv)
try:
db.session.commit()
print "Result property values calculated."
except Exception as e:
print "Error while saving result property values"
db.session.rollback()
def phase(database):
if request.method == 'POST':
competition = request.form.get('competition', None)
competition_phase = int(request.form.get('competition_phase', 0))
models.get_database(database).set_competition(1 if competition == 'on' else 0)
models.get_database(database).set_competition_phase(competition_phase)
models.get_database(database).session.commit()
competition = models.get_database(database).is_competition()
competition_phase = models.get_database(database).competition_phase()
return render('/admin/phase.html', database=database, phases=range(1, 8),
competition=competition, competition_phase=competition_phase)
def admin_toggle_solver_freeze(database, solver_id):
db = models.get_database(database) or abort(404)
solver = db.session.query(db.Solver).get(solver_id) or abort(404)
solver.competition_frozen = not solver.competition_frozen
try:
db.session.commit()
except:
db.session.rollback()
flash('DB error when commiting. Rolled back.')
return redirect(url_for('frontend.solver_details',
database=database, solver_id=solver_id))
def action(db_name=('db','EDACC'), experiment=('experiment', '')):
"""Calculate result properties for the jobs of the experiment with the name passed in."""
db = models.get_database(db_name)
exp = db.session.query(db.Experiment).filter_by(name=experiment).first()
if exp is None:
print "Experiment doesn't exist"
return
props = db.session.query(db.Property).filter_by(propertyType=1)
for p in props:
pat = re.compile(p.regExpression)
for result in db.session.query(db.ExperimentResult).filter(db.ExperimentResult.resultCode.like('1%')).filter_by(experiment=exp).all():
for pv in result.properties:
if pv.property == p:
for pvv in pv.values:
db.session.delete(pvv)
db.session.commit()
db.session.delete(pv)
db.session.commit()
if p.propertySource == 2:
output = result.launcherOutput
elif p.propertySource == 4:
output = result.solverOutput
elif p.propertySource == 5:
output = result.verifierOutput
elif p.propertySource == 6:
output = result.watcherOutput
m = re.search(pat, output)
if m is not None:
val = m.groups()[-1]
pv = db.ExperimentResultProperty()
pv.property = p
pv.experiment_result = result
db.session.add(pv)
db.session.flush()
pvv = db.ResultPropertyValue()
pvv.value = val
pvv.order = 0
pvv.experiment_result_property = pv
db.session.add(pvv)
try:
db.session.commit()
print "Result property values calculated."
except Exception as e:
print "Error while saving result property values"
db.session.rollback()
def reset_password(database):
db = models.get_database(database) or abort(404)
form = forms.ResetPasswordForm()
if form.validate_on_submit():
# find user by lower case email address
user = db.session.query(
db.User).filter_by(email=form.email.data.lower()).first()
if not user or not user.verified:
if not user: flash('No account with this e-mail address exists.')
if user and not user.verified:
flash('Account was not verified yet.')
return render('/accounts/reset_password.html',
db=db,
database=database,
form=form)
hash = hashlib.sha256()
hash.update(config.SECRET_KEY)
hash.update(user.email)
hash.update(str(datetime.datetime.now()))
hash.update(user.password)
# reuse the activation hash (user should be activated at this point already)
user.activation_hash = 'pw_reset_' + hash.hexdigest()
try:
db.session.commit()
msg = Message("[" + db.label + "] Password reset instructions",
recipients=[user.email])
msg.body = "Dear " + user.firstname + " " + user.lastname + ",\n\n" + \
"If you did not use the password reset link on the website ignore this mail.\n\n" + \
"To reset your password please use the following link:\n" + \
request.url_root[:-1] + url_for('accounts.change_password', database=database,
reset_hash=hash.hexdigest())
mail.send(msg)
flash(
'E-mail was sent. Please refer to the mail for further instructions.'
)
except Exception as e:
print e
flash(
'Could not send reset mail. Please contact an administrator.')
return render('/accounts/reset_password.html',
db=db,
database=database,
form=form)
def submit_benchmarks(database):
db = models.get_database(database) or abort(404)
form = forms.BenchmarksForm()
if form.validate_on_submit():
upload_dir = os.path.join(config.UPLOAD_FOLDER, database,
str(g.User.idUser))
move_dir = os.path.join(config.UPLOAD_FOLDER, database,
secure_filename(form.category.data),
str(g.User.idUser))
try:
os.makedirs(upload_dir, mode=0700)
except:
pass
try:
os.makedirs(move_dir, mode=0700)
except:
pass
try:
for file in session.get('benchmarks', list()):
try:
os.rename(os.path.join(upload_dir, file),
os.path.join(move_dir, file))
except Exception as ex:
print ex
flash('Benchmark submission successful.')
try:
msg = Message("[" + db.label + "][Admin] Benchmarks submitted",
recipients=[config.DEFAULT_MAIL_SENDER])
msg.body = (
"The user %s %s with id %d just submitted some benchmarks"
% (g.User.firstname, g.User.lastname, g.User.idUser))
mail.send(msg)
except:
pass
session.pop('benchmarks', None)
return redirect(
url_for('frontend.experiments_index', database=database))
except Exception as ex:
print ex
flash('Error occured when trying to move the uploaded files.')
return render('/accounts/submit_benchmarks.html',
db=db,
database=database,
form=form)
def admin_toggle_solver_freeze(database, solver_id):
db = models.get_database(database) or abort(404)
solver = db.session.query(db.Solver).get(solver_id) or abort(404)
solver.competition_frozen = not solver.competition_frozen
try:
db.session.commit()
except:
db.session.rollback()
flash('DB error when commiting. Rolled back.')
return redirect(
url_for('frontend.solver_details',
database=database,
solver_id=solver_id))
def list_solvers(database, user_id=None):
""" Lists all solvers that the user submitted to
the database
"""
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).get(user_id or g.User.idUser) or abort(404)
if is_admin() and user_id:
solvers = db.session.query(db.Solver).filter_by(User_idUser=user_id).all()
else:
if user_id and g.User.idUser != user_id: abort(401)
solvers = db.session.query(db.Solver).filter_by(user=g.User).all()
return render('/accounts/list_solvers.html', database=database, user=user,
solvers=solvers, db=db)
def phase(database):
if request.method == 'POST':
competition = request.form.get('competition', None)
competition_phase = int(request.form.get('competition_phase', 0))
models.get_database(database).set_competition(1 if competition ==
'on' else 0)
models.get_database(database).set_competition_phase(competition_phase)
models.get_database(database).session.commit()
competition = models.get_database(database).is_competition()
competition_phase = models.get_database(database).competition_phase()
return render('/admin/phase.html',
database=database,
phases=range(1, 8),
competition=competition,
competition_phase=competition_phase)
def upload_benchmarks(database):
db = models.get_database(database) or abort(404)
if request.files:
upload_dir = os.path.join(config.UPLOAD_FOLDER, database, str(g.User.idUser))
filename = secure_filename(request.files['file'].filename)
# save list of uploaded files in user session so we can track them until form submission
benchmarks = session.get('benchmarks', list())
benchmarks.append(filename)
session['benchmarks'] = benchmarks
try:
os.makedirs(upload_dir, mode=0700)
except:
pass
request.files['file'].save(os.path.join(upload_dir, filename))
return 'success'
def login(database):
""" User login form and handling for a specific database. Users can
only be logged in to one database at a time
"""
db = models.get_database(database) or abort(404)
form = forms.LoginForm(csrf_enabled=False)
error = None
if form.validate_on_submit():
user = db.session.query(
db.User).filter_by(email=form.email.data).first()
if user is None:
error = "Invalid password or username."
else:
if user.activation_hash:
error = "Account not activated yet. Please check your e-mail account, otherwise contact an administrator."
elif not user.verified:
error = "Account was not verified yet. Please wait for an administrator to verify your account. You will be notified by e-mail."
elif user.password != password_hash(form.password.data):
error = 'Invalid password or username.'
else:
session['logged_in'] = True
session['database'] = database
session['idUser'] = user.idUser
session['email'] = user.email
session['db'] = str(db)
session['admin'] = user.admin
session.permanent = form.permanent_login.data
# if db.is_competition() and db.competition_phase() == 5:
# if not user.admin:
# session.pop('logged_in', None)
# flash('Website offline for competition computations.')
# return redirect(url_for('frontend.experiments_index',
# database=database))
flash('Login successful')
return redirect(
url_for('frontend.experiments_index', database=database))
return render('/accounts/login.html',
database=database,
error=error,
db=db,
form=form)
def configuration_runs(database, experiment_id):
db = models.get_database(database) or abort(404)
experiment = db.session.query(db.Experiment).get(experiment_id) or abort(404)
if not experiment.configuration_scenario: abort(404)
solver_configs = [sc for sc in experiment.solver_configurations if
sc.solver_binary == experiment.configuration_scenario.solver_binary]
solver_config_ids = [sc.idSolverConfig for sc in solver_configs]
configurable_parameters = [p.parameter for p in experiment.configuration_scenario.parameters if
p.configurable and p.parameter.name not in ('instance', 'seed')]
configurable_parameters_ids = [p.idParameter for p in configurable_parameters]
parameter_instances = db.session.query(db.ParameterInstance).options(joinedload('parameter')).filter(
db.ParameterInstance.SolverConfig_idSolverConfig.in_(solver_config_ids)).all()
instances_by_id = dict((i.idInstance, i) for i in experiment.get_instances(db))
instance_properties = [p for p in db.session.query(db.Property) if p.is_instance_property()]
parameter_values = dict()
for pv in parameter_instances:
if pv.Parameters_idParameter not in configurable_parameters_ids: continue
if pv.SolverConfig_idSolverConfig not in parameter_values:
parameter_values[pv.SolverConfig_idSolverConfig] = dict()
parameter_values[pv.SolverConfig_idSolverConfig][pv.Parameters_idParameter] = pv.value
results, _, _ = experiment.get_result_matrix(db, experiment.solver_configurations, experiment.get_instances(db),
cost=experiment.defaultCost)
csv_response = StringIO.StringIO()
csv_writer = csv.writer(csv_response)
csv_writer.writerow(
[p.name for p in configurable_parameters] + [p.name for p in instance_properties] + ['par1', 'censored'])
for idInstance in results:
for idSolverConfig in results[idInstance]:
for run in results[idInstance][idSolverConfig]:
csv_writer.writerow(
[parameter_values[idSolverConfig].get(p.idParameter, '') for p in configurable_parameters] + \
[instances_by_id[idInstance].get_property_value(p.idProperty, db) for p in instance_properties] + \
[run.penalized_time1, 1 if run.censored else 0])
csv_response.seek(0)
headers = Headers()
headers.add('Content-Type', 'text/csv')
headers.add('Content-Disposition', 'attachment',
filename=secure_filename(experiment.name) + "_configuration_runs.csv")
return Response(response=csv_response.read(), headers=headers)
def admin_list_benchmarks(database):
""" Lists all benchmarks that the currently logged in user submitted.
"""
db = models.get_database(database) or abort(404)
uploaded_files = {}
directory = os.path.join(config.UPLOAD_FOLDER, database)
for file in os.listdir(directory):
if os.path.isdir(os.path.join(directory, file)):
for user_dir in os.listdir(os.path.join(directory, file)):
if not file in uploaded_files: uploaded_files[file] = list()
files = os.listdir(os.path.join(directory, file, user_dir))
files_full_path = [os.path.join(directory, file, user_dir, f) for f in files]
files_sizes = [os.stat(f).st_size for f in files_full_path]
uploaded_files[file] += zip(files, [time.ctime(os.path.getmtime(f)) for f in files_full_path],
[user_dir] * len(files), files_sizes)
return render('/accounts/admin_list_benchmarks.html', database=database,
db=db, uploaded_files=uploaded_files)
def update_description(database, solver_id):
db = models.get_database(database) or abort(404)
solver = db.session.query(db.Solver).get(solver_id) or abort(404)
form = forms.UpdateDescriptionForm()
if form.validate_on_submit():
solver.description_pdf = request.files[form.description_pdf.name].stream.read()
try:
db.session.commit()
flash("Solver description updated.")
return redirect(url_for('accounts.list_submitted_solvers',
database=database))
except Exception as e:
db.session.rollback()
flash("Error updating description: " + str(e))
return render("/accounts/update_description.html", database=database, db=db, solver=solver, form=form)
def upload_benchmarks(database):
db = models.get_database(database) or abort(404)
if request.files:
upload_dir = os.path.join(config.UPLOAD_FOLDER, database,
str(g.User.idUser))
filename = secure_filename(request.files['file'].filename)
# save list of uploaded files in user session so we can track them until form submission
benchmarks = session.get('benchmarks', list())
benchmarks.append(filename)
session['benchmarks'] = benchmarks
try:
os.makedirs(upload_dir, mode=0700)
except:
pass
request.files['file'].save(os.path.join(upload_dir, filename))
return 'success'
def list_benchmarks(database, user_id=None):
""" Lists all benchmarks that the currently logged in user submitted.
"""
db = models.get_database(database) or abort(404)
uploaded_files = {}
directory = os.path.join(config.UPLOAD_FOLDER, database)
for file in os.listdir(directory):
if os.path.isdir(os.path.join(directory, file)):
for user_dir in os.listdir(os.path.join(directory, file)):
if (is_admin() and user_id and user_dir == str(user_id)) or (
not user_id and user_dir == str(g.User.idUser)):
if not file in uploaded_files: uploaded_files[file] = list()
files = os.listdir(os.path.join(directory, file, user_dir))
files_full_path = [os.path.join(directory, file, user_dir, f) for f in files]
uploaded_files[file] += zip(files, [time.ctime(os.path.getmtime(f)) for f in files_full_path])
return render('/accounts/list_benchmarks.html', database=database,
db=db, uploaded_files=uploaded_files, user_id=user_id)
def list_solvers(database, user_id=None):
""" Lists all solvers that the user submitted to
the database
"""
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).get(user_id
or g.User.idUser) or abort(404)
if is_admin() and user_id:
solvers = db.session.query(
db.Solver).filter_by(User_idUser=user_id).all()
else:
if user_id and g.User.idUser != user_id: abort(401)
solvers = db.session.query(db.Solver).filter_by(user=g.User).all()
return render('/accounts/list_solvers.html',
database=database,
user=user,
solvers=solvers,
db=db)
def databases_add():
""" Display a form to add databases to the web frontend """
error = None
if request.method == 'POST':
label = request.form['label']
database = request.form['database']
username = request.form['username']
password = request.form['password']
hidden = request.form.has_key('hidden')
if models.get_database(database):
error = "A database with this name already exists"
else:
try:
models.add_database(username, password, database, label, hidden)
return redirect(url_for('frontend.index'))
except Exception as e:
error = "Can't add database: " + str(e)
return render('/admin/databases_add.html', error=error)
def change_password(database, reset_hash):
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).filter_by(activation_hash='pw_reset_' + reset_hash).first() or abort(404)
form = forms.ChangePasswordForm()
if form.validate_on_submit():
user.activation_hash = ''
user.password = password_hash(form.password.data)
try:
db.session.commit()
flash('Password changed.')
except Exception as e:
print e
flash('Could not set password in db. Please contact an administrator.')
return redirect(url_for('frontend.experiments_index',
database=database))
return render('/accounts/change_password.html', db=db, database=database, form=form, reset_hash=reset_hash)
def login(database):
""" User login form and handling for a specific database. Users can
only be logged in to one database at a time
"""
db = models.get_database(database) or abort(404)
form = forms.LoginForm(csrf_enabled=False)
error = None
if form.validate_on_submit():
user = db.session.query(db.User).filter_by(email=form.email.data).first()
if user is None:
error = "Invalid password or username."
else:
if user.activation_hash:
error = "Account not activated yet. Please check your e-mail account, otherwise contact an administrator."
elif not user.verified:
error = "Account was not verified yet. Please wait for an administrator to verify your account. You will be notified by e-mail."
elif user.password != password_hash(form.password.data):
error = 'Invalid password or username.'
else:
session['logged_in'] = True
session['database'] = database
session['idUser'] = user.idUser
session['email'] = user.email
session['db'] = str(db)
session['admin'] = user.admin
session.permanent = form.permanent_login.data
# if db.is_competition() and db.competition_phase() == 5:
# if not user.admin:
# session.pop('logged_in', None)
# flash('Website offline for competition computations.')
# return redirect(url_for('frontend.experiments_index',
# database=database))
flash('Login successful')
return redirect(url_for('frontend.experiments_index',
database=database))
return render('/accounts/login.html', database=database, error=error,
db=db, form=form)
def databases_add():
""" Display a form to add databases to the web frontend """
error = None
if request.method == 'POST':
label = request.form['label']
database = request.form['database']
username = request.form['username']
password = request.form['password']
hidden = request.form.has_key('hidden')
if models.get_database(database):
error = "A database with this name already exists"
else:
try:
models.add_database(username, password, database, label,
hidden)
return redirect(url_for('frontend.index'))
except Exception as e:
error = "Can't add database: " + str(e)
return render('/admin/databases_add.html', error=error)
def submit_benchmarks(database):
db = models.get_database(database) or abort(404)
form = forms.BenchmarksForm()
if form.validate_on_submit():
upload_dir = os.path.join(config.UPLOAD_FOLDER, database, str(g.User.idUser))
move_dir = os.path.join(config.UPLOAD_FOLDER, database, secure_filename(form.category.data), str(g.User.idUser))
try:
os.makedirs(upload_dir, mode=0700)
except:
pass
try:
os.makedirs(move_dir, mode=0700)
except:
pass
try:
for file in session.get('benchmarks', list()):
try:
os.rename(os.path.join(upload_dir, file), os.path.join(move_dir, file))
except Exception as ex:
print ex
flash('Benchmark submission successful.')
try:
msg = Message("[" + db.label + "][Admin] Benchmarks submitted",
recipients=[config.DEFAULT_MAIL_SENDER])
msg.body = ("The user %s %s with id %d just submitted some benchmarks" % (
g.User.firstname, g.User.lastname, g.User.idUser))
mail.send(msg)
except:
pass
session.pop('benchmarks', None)
return redirect(url_for('frontend.experiments_index', database=database))
except Exception as ex:
print ex
flash('Error occured when trying to move the uploaded files.')
return render('/accounts/submit_benchmarks.html', db=db, database=database, form=form)
def borg_explorer_data(database, experiment_id):
db = models.get_database(database) or abort(404)
experiment = db.session.query(
db.Experiment).get(experiment_id) or abort(404)
type = request.args.get('type')
if type == 'categories.json':
return json_dumps([{"path": experiment.name, "name": experiment.name}])
last_modified_job = db.session.query(func.max(db.ExperimentResult.date_modified))\
.filter_by(experiment=experiment).first()
job_count = db.session.query(
db.ExperimentResult).filter_by(experiment=experiment).count()
@synchronized
@cache.memoize(7 * 24 * 60 * 60)
def get_data(database, experiment_id, job_count, last_modified_job):
runs = db.session.query(db.ExperimentResult) \
.filter(db.ExperimentResult.Experiment_idExperiment==experiment_id) \
.filter(not_(db.ExperimentResult.status.in_(STATUS_PROCESSING))).order_by('idJob').all()
return CategoryData().fit([
(0, r.instance.name, r.result_code.description, r.resultTime, 0,
r.solver_configuration.name, 0) for r in runs
])
data = get_data(database, experiment_id, job_count, last_modified_job)
if type == 'runs.json':
return json_dumps(data.table)
elif type == 'solvers.json':
return json_dumps(data.solvers)
elif type == 'instances.json':
return json_dumps(data.instances)
elif type == 'membership.json':
return json_dumps(data.model._tclass_res_LN.T.tolist())
elif type == 'projection.json':
return json_dumps(data.projection_N2.tolist())
else:
abort(404)
def reset_password(database):
db = models.get_database(database) or abort(404)
form = forms.ResetPasswordForm()
if form.validate_on_submit():
# find user by lower case email address
user = db.session.query(db.User).filter_by(email=form.email.data.lower()).first()
if not user or not user.verified:
if not user: flash('No account with this e-mail address exists.')
if user and not user.verified: flash('Account was not verified yet.')
return render('/accounts/reset_password.html', db=db, database=database, form=form)
hash = hashlib.sha256()
hash.update(config.SECRET_KEY)
hash.update(user.email)
hash.update(str(datetime.datetime.now()))
hash.update(user.password)
# reuse the activation hash (user should be activated at this point already)
user.activation_hash = 'pw_reset_' + hash.hexdigest()
try:
db.session.commit()
msg = Message("[" + db.label + "] Password reset instructions",
recipients=[user.email])
msg.body = "Dear " + user.firstname + " " + user.lastname + ",\n\n" + \
"If you did not use the password reset link on the website ignore this mail.\n\n" + \
"To reset your password please use the following link:\n" + \
request.url_root[:-1] + url_for('accounts.change_password', database=database,
reset_hash=hash.hexdigest())
mail.send(msg)
flash('E-mail was sent. Please refer to the mail for further instructions.')
except Exception as e:
print e
flash('Could not send reset mail. Please contact an administrator.')
return render('/accounts/reset_password.html', db=db, database=database, form=form)
def update_description(database, solver_id):
db = models.get_database(database) or abort(404)
solver = db.session.query(db.Solver).get(solver_id) or abort(404)
form = forms.UpdateDescriptionForm()
if form.validate_on_submit():
solver.description_pdf = request.files[
form.description_pdf.name].stream.read()
try:
db.session.commit()
flash("Solver description updated.")
return redirect(
url_for('accounts.list_submitted_solvers', database=database))
except Exception as e:
db.session.rollback()
flash("Error updating description: " + str(e))
return render("/accounts/update_description.html",
database=database,
db=db,
solver=solver,
form=form)
def delete_solver(database, solver_id):
db = models.get_database(database) or abort(404)
solver = db.session.query(db.Solver).get(solver_id) or abort(404)
if solver.user.idUser != g.User.idUser: abort(401)
try:
for solver_binary in solver.binaries:
for solver_config in solver_binary.solver_configurations:
for pi in solver_config.parameter_instances: db.session.delete(pi)
db.session.commit()
db.session.delete(solver_config)
db.session.commit()
db.session.delete(solver_binary)
for p in solver.parameters: db.session.delete(p)
db.session.delete(solver)
db.session.commit()
flash('Solver deleted successfully.')
except Exception as e:
print e
db.session.rollback()
flash('Could not delete solver. Please contact an administrator.')
return redirect(url_for('accounts.list_solvers', database=database, user_id=None))
def action(db_name=('db', 'EDACC'),
propertyType=('type', 0),
propertySource=('source', 0),
name=('name', ''),
description=('desc', ''),
regExp=('regExp', ''),
value_type=('value_type', ''),
multiple=('multiple', 0)):
"""Add a new property to the database.
propertyType: Instance = 0, Result = 1
propertySource: Instance = 0, InstanceName = 1, LauncherOutput = 2, Parameter = 3, SolverOutput = 4, VerifierOutput = 5, WatcherOutput = 6
value_type: name of a PropertyValueType
multiple: multiple occurences 1/0 for true/false
Arguments in order:
"""
db = models.get_database(db_name)
p = db.session.query(db.Property).filter_by(name=name).first()
if p is None:
p = db.Property()
p.propertyType = int(propertyType)
p.propertySource = int(propertySource)
p.name = name
p.description = description
p.regExpression = regExp
p.propertyValueType = value_type
p.isDefault = True
p.multipleOccourence = int(multiple)
db.session.add(p)
try:
db.session.commit()
print "Property saved."
except Exception as e:
print "Couldn't save property to database:", e
db.session.rollback()
def list_solver_descriptions(database):
db = models.get_database(database) or abort(404)
solvers = [s for s in db.session.query(db.Solver).all() if s.User_idUser]
filtered_solvers = []
for s in solvers:
if not any(ss.description_pdf == s.description_pdf and s.description_pdf for ss in filtered_solvers):
filtered_solvers.append(s)
solvers_by_category = dict()
filtered_solvers_by_category = dict()
for category in db.session.query(db.CompetitionCategory).all():
solvers_by_category[category] = [s for s in category.solvers if s.User_idUser]
filtered_solvers_by_category[category] = []
for s in solvers_by_category[category]:
if not any(ss.description_pdf == s.description_pdf and s.description_pdf for ss in
filtered_solvers_by_category[category]):
filtered_solvers_by_category[category].append(s)
return render('/accounts/list_solver_descriptions.html', database=database,
categories=sorted(solvers_by_category.keys(), key=lambda c: c.name),
db=db, solvers=solvers, solvers_by_category=solvers_by_category, sorted=sorted,
filtered_solvers=filtered_solvers,
filtered_solvers_by_category=filtered_solvers_by_category)
def decorated_f(*args, **kwargs):
db = models.get_database(kwargs['database']) or abort(404)
if session.get('logged_in'):
g.User = db.session.query(db.User).get(session['idUser'])
if g.User.admin: session['admin'] = True
else:
g.User = None
if db.is_competition() and db.competition_phase() == 7:
session.pop('logged_in', None)
session.pop('idUser', None)
g.User = None
# if db.is_competition() and db.competition_phase() == 5:
# def redirect_f(*args, **kwargs):
# return redirect(url_for('frontend.experiments_index',
# database=kwargs['database']))
# if not g.User or not g.User.admin:
# session.pop('logged_in', None)
# flash('Website offline for competition computations.')
# return redirect_f(*args, **kwargs)
if db.is_competition() and db.competition_phase() < 7:
def redirect_f(*args, **kwargs):
return redirect(
url_for('accounts.login', database=kwargs['database']))
if not g.User or not session.get('logged_in') or \
session.get('idUser', None) is None:
return redirect_f(*args, **kwargs)
if session.get('database') != kwargs['database']:
return redirect_f(*args, **kwargs)
return f(*args, **kwargs)
def verify_user(database, user_id):
db = models.get_database(database) or abort(404)
user = db.session.query(db.User).get(user_id) or abort(404)
if user.verified:
flash('User already verified.')
return redirect(url_for('frontend.experiments_index',
database=database))
user.verified = True
try:
db.session.commit()
msg = Message("[" + db.label + "] Account verified",
recipients=[user.email])
msg.body = "Dear " + user.firstname + " " + user.lastname + ",\n\n" + \
"Your account was verified and you can now log in:\n" + \
request.url_root[:-1] + url_for('accounts.login', database=database)
mail.send(msg)
flash('Verified the account.')
except Exception as e:
db.session.rollback()
flash("Couldn't update verification status of user or send the notification mail: " + str(e))
return redirect(url_for('frontend.experiments_index',
database=database))
def admin_list_benchmarks(database):
""" Lists all benchmarks that the currently logged in user submitted.
"""
db = models.get_database(database) or abort(404)
uploaded_files = {}
directory = os.path.join(config.UPLOAD_FOLDER, database)
for file in os.listdir(directory):
if os.path.isdir(os.path.join(directory, file)):
for user_dir in os.listdir(os.path.join(directory, file)):
if not file in uploaded_files: uploaded_files[file] = list()
files = os.listdir(os.path.join(directory, file, user_dir))
files_full_path = [
os.path.join(directory, file, user_dir, f) for f in files
]
files_sizes = [os.stat(f).st_size for f in files_full_path]
uploaded_files[file] += zip(
files,
[time.ctime(os.path.getmtime(f)) for f in files_full_path],
[user_dir] * len(files), files_sizes)
return render('/accounts/admin_list_benchmarks.html',
database=database,
db=db,
uploaded_files=uploaded_files)
def borg_explorer(database, experiment_id):
db = models.get_database(database) or abort(404)
experiment = db.session.query(db.Experiment).get(experiment_id) or abort(404)
return render('borgexplorer/index.html', db=db, database=database, experiment=experiment)
""" Parse solver info from the SAT Competition 09 website.
To be used after the import script.
"""
import sys, urllib2
sys.path.append("..")
from BeautifulSoup import BeautifulSoup
from edacc import models, config
config.DATABASE_HOST = "localhost"
models.add_database("edacc", "passw", "EDACC5", "EDACC5")
db = models.get_database("EDACC5")
html_url = raw_input(
'Enter solverlist url (e.g. http://www.cril.univ-artois.fr/SAT09/results/solverlist.php?idev=22): '
)
p = urllib2.urlopen(html_url)
soup = BeautifulSoup(p.read())
for solver_tab in soup.findAll('table'):
if not solver_tab.find('td', text='Event'): continue
version = solver_tab.find('td',
attrs={
"class": "fieldname"
},
text='Version').findNext('td').string.strip()
name = solver_tab.find('td', attrs={
"class": "fieldname"
}, text='Name').findNext('td').string.strip()
authors = solver_tab.find('td',
attrs={
"class": "fieldname"
def list_submitted_solvers(database):
db = models.get_database(database) or abort(404)
solvers = db.session.query(db.Solver).filter(db.Solver.User_idUser != None).order_by(db.Solver.name).all()
return render("/accounts/list_submitted_solvers.html", database=database, db=db, solvers=solvers)
def __init__(self, database, expID, configForm, standardize):
db = models.get_database(database) or abort(404)
experiment = db.session.query(db.Experiment).get(expID) or abort(404)
if experiment.configurationExp == False:
return # kein Konfiguratorexperiment
solverConfig = {}
paramAttribute = {}
parameterValue = {}
parameterName = {}
paramList = []
numValue = 0
minDict = {}
maxDict = {}
selectValueList = {}
deselectedConfigs = []
choosenConfigs = []
turnList = []
hideList = []
configList = []
domain = {}
parameterDomain = {}
parameterPosition = {}
page = 0
absMax = 0
negNum = 0
#db Queries for configuration visualisation
name = db.session.query(db.Experiment.name).filter(
db.Experiment.idExperiment == expID).first()
configuration['expName'] = str(name[0])
configuration['standardize'] = standardize
if configForm == None:
configuration['page'] = page
else:
page = map(int, configForm.getlist("page"))[0] + 1
configuration['page'] = page
minDict['confidence'] = map(
str, configForm.getlist("min_confidence"))[0]
maxDict['confidence'] = map(
str, configForm.getlist("max_confidence"))[0]
minDict['performance'] = map(
str, configForm.getlist("min_performance"))[0]
maxDict['performance'] = map(
str, configForm.getlist("max_performance"))[0]
paramList.append('confidence')
paramList.append('performance')
parameterDomain['confidence'] = "integerDomain"
parameterDomain['performance'] = "realDomain"
solverConfigCosts = dict((s.idSolverConfig, s.cost)
for s in experiment.solver_configurations)
solverConfigName = dict(
(s.idSolverConfig, s.name)
for s in db.session.query(db.SolverConfiguration).filter(
db.SolverConfiguration.Experiment_idExperiment == expID))
if page == 0:
choosenConfigs = solverConfigName.keys()
for scn in solverConfigName:
count = int(
db.session.query(db.ExperimentResult.idJob).filter(
db.ExperimentResult.SolverConfig_idSolverConfig ==
scn).count())
parameterValue[scn] = {'confidence': count}
if solverConfigCosts[scn] != None:
parameterValue[scn].update(
{'performance': solverConfigCosts[scn]})
else:
parameterValue[scn].update({'performance': 0.0})
#deselect solverConfigs which aren't in the range of choosen confidence or performance
if page > 0:
if count >= int(minDict['confidence']) and count <= int(
maxDict['confidence']
) and float(parameterValue[scn]['performance']) >= float(
minDict['performance']) and float(
parameterValue[scn]['performance']) <= float(
maxDict['performance']):
choosenConfigs.append(scn)
else:
deselectedConfigs.append(scn)
if page > 0:
configurable = [
p.Parameters_idParameter
for p in experiment.configuration_scenario.parameters
if p.configurable and p.parameter.name not in ('instance',
'seed')
]
parameterName = dict(
(p.idParameter, p.name)
for p in db.session.query(db.Parameter).filter(
db.Parameter.idParameter.in_(configurable)).distinct())
for id in parameterName.keys():
parameterDomain[
id] = experiment.configuration_scenario.get_parameter_domain(
parameterName[id])
paramInstance = db.session.query(db.ParameterInstance).filter(
db.ParameterInstance.SolverConfig_idSolverConfig.in_(
choosenConfigs)).all()
for pv in paramInstance:
if pv.Parameters_idParameter not in parameterName.keys(
) or pv.value == "":
continue
if pv.SolverConfig_idSolverConfig not in parameterValue:
parameterValue[pv.SolverConfig_idSolverConfig] = {}
parameterValue[pv.SolverConfig_idSolverConfig].update(
{pv.Parameters_idParameter: pv.value})
if pv.Parameters_idParameter not in paramList:
paramList.append(pv.Parameters_idParameter)
parameterName.update({
'confidence': 'confidence',
'performance': 'performance'
})
for pd in parameterDomain.keys():
selectValueList[pd] = []
if parameterDomain[pd] == "realDomain" or parameterDomain[
pd] == "integerDomain":
domain[pd] = 'num'
else:
domain[pd] = 'cat'
#maps the web formular in lists
if page > 1:
for pm in paramList:
if str(pm) in configForm.keys():
tmpList = mapPosition(configForm.getlist(str(pm)))
parameterPosition[tmpList[0]] = [pm, tmpList[1]]
if domain[pm] == "num":
indexMin = "min_" + str(pm)
minList = []
if indexMin in configForm.keys():
minList = map(str, configForm.getlist(indexMin))
minDict[pm] = minList[0].strip()
indexMax = "max_" + str(pm)
maxList = []
if indexMax in configForm.keys():
maxList = map(str, configForm.getlist(indexMax))
maxDict[pm] = maxList[0].strip()
elif domain[pm] == "cat":
index = "select_" + str(pm)
if index in configForm.keys():
select = map(str, configForm.getlist(index))
selectValueList[pm] = select
if 'turn' in configForm.keys():
turnList = map(str, configForm.getlist('turn'))
if 'hide' in configForm.keys():
hideList = map(str, configForm.getlist('hide'))
if 'solverConfigs' in configForm.keys():
configList = map(str, configForm.getlist('solverConfigs'))
#creates a dictionary with values of the parameters of each solverConfig
for scn in choosenConfigs:
parameterInstance = {}
parameter = {}
parameterInstance['name'] = str(solverConfigName[scn])
n = 2
for p in parameterValue[scn]:
#assigns the value to the parameter of the solverConfig
parameter[p] = str(parameterValue[scn][p])
#creates the list deselectedConfigs of solverConfigs which are deselected if the values are restricted
if page > 1 and domain[p] == 'cat':
if "select_" + str(p) in configForm.keys():
if str(parameterValue[scn]
[p]) not in selectValueList[p]:
deselectedConfigs.append(scn)
elif page > 1 and domain[p] == 'num':
if len(minDict[p]) < 0:
if float(parameterValue[scn][p]) < float(minDict[p]):
deselectedConfigs.append(scn)
if len(maxDict[p]) < 0:
if float(parameterValue[scn][p]) > float(maxDict[p]):
deselectedConfigs.append(scn)
n += 1
parameterInstance['parameter'] = parameter
solverConfig[scn] = parameterInstance
#chance the position
if page > 1:
formerPosition = []
for pp in parameterPosition:
#create a list with parameter id in the former order
formerPosition.append(parameterPosition[pp][0])
for pp in parameterPosition:
requestedPosition = parameterPosition[pp][1]
paramID = parameterPosition[pp][0]
if pp != requestedPosition:
del formerPosition[formerPosition.index(paramID)]
formerPosition.insert((requestedPosition - 1), paramID)
paramList = formerPosition[:]
i = 0
for pl in paramList:
values = []
valueList = []
expectedValue = 0.0
variance = 0.0
minValue = 0
maxValue = 0
turn = False
i += 1
#creates a list of possible values (valueList) and an list of values for each parameter
for scn in choosenConfigs:
if (pl in solverConfig[scn]['parameter']):
tmp = solverConfig[scn]['parameter'][pl]
if (tmp not in valueList):
valueList.append(tmp)
if scn not in deselectedConfigs:
values.append(tmp)
if (domain[pl] == 'num'):
if standardize == 1:
expectedValue = expectedValue + float(tmp)
else:
if scn not in deselectedConfigs:
values.append(0)
if len(valueList) == 0:
valueList.append(0)
values.append(0)
#maps the values of each parameter suitable for domains
if domain[pl] == "cat":
values = classify(values, valueList)
elif domain[pl] == "num":
iv = 0
for v in values:
try:
values[iv] = float(v)
iv += 1
except:
values[iv] = 0.0
iv += 1
ivl = 0
for vl in valueList:
try:
valueList[ivl] = float(vl)
ivl += 1
except:
valueList[ivl] = 0.0
ivl += 1
if len(turnList) > 0 and (str(pl) in turnList):
turn = True
values = turnValue(values)
#checks if a parameter is hidden
hide = False
if len(hideList) > 0 and (str(pl) in hideList):
hide = True
positionList = []
length = len(paramList)
for p in range(length):
pos = (p + i - 1) % length
positionList.append(pos + 1)
if numValue < len(values):
numValue = len(values)
if domain[pl] == 'num':
if page > 1 and len(minDict[pl]) > 0 and float(
minDict[pl]) >= min(valueList) and float(
minDict[pl]) <= max(valueList):
minValue = minDict[pl]
elif len(choosenConfigs) == 0:
minValue = minDict[pl]
else:
minValue = min(valueList)
if page > 1 and len(maxDict[pl]) > 0 and float(
maxDict[pl]) >= min(valueList) and float(
maxDict[pl]) <= max(valueList):
maxValue = maxDict[pl]
elif len(choosenConfigs) == 0:
maxValue = maxDict[pl]
else:
maxValue = max(valueList)
if (parameterDomain[pl] == "integerDomain"):
minValue = int(minValue)
maxValue = int(maxValue)
if standardize == 1:
negNum = 1
expectedValue = expectedValue / len(values)
for v in values:
variance = variance + (v - expectedValue)**2
variance = math.sqrt(variance / len(values))
standardScore = lambda x: (
x - expectedValue) / variance if variance != 0 else 0
values = map(standardScore, values)
else:
if min(values) < 0:
negNum = 1
if math.fabs(min(values)) > max(values):
values = [
(v - min(values)) / max(values) for v in values
] # project(values, math.fabs(min(values)))
else:
values = [
(v - min(values)) / max(values) for v in values
] # project(values, max(values))
if max(values) > absMax:
absMax = max(values)
if math.fabs(min(values)) > absMax:
absMax = math.fabs(min(values))
paramAttribute[i] = {
'values': values,
'min': minValue,
'max': maxValue,
'name': parameterName[pl],
'id': pl,
'hide': hide,
'turn': turn,
'positionList': positionList,
'domain': 'num'
}
elif domain[pl] == 'cat':
values = project(values, max(values))
paramAttribute[i] = {
'values': values,
'valueList': valueList,
'selectValueList': selectValueList[pl],
'name': parameterName[pl],
'id': pl,
'hide': hide,
'turn': turn,
'positionList': positionList,
'domain': 'cat'
}
if standardize == 1:
for ri in range(i):
if paramAttribute[ri + 1]['domain'] == 'num':
paramAttribute[ri + 1]['values'] = project(
paramAttribute[ri + 1]['values'], absMax)
configuration['paramAttribute'] = paramAttribute
list = []
for i in range(numValue):
list.append(i)
configuration['numValue'] = list
configuration['negNum'] = negNum
if page > 1:
selectedConfigs = []
for scn in choosenConfigs:
if scn not in deselectedConfigs:
if str(scn) in configList:
selectedConfigs.append([scn, solverConfigName[scn], 1])
else:
selectedConfigs.append([scn, solverConfigName[scn], 0])
configuration['solverConfigs'] = selectedConfigs
else:
selectedConfigs = []
for scn in choosenConfigs:
selectedConfigs.append([scn, solverConfigName[scn], 0])
configuration['solverConfigs'] = selectedConfigs
def configuration_runs(database, experiment_id):
db = models.get_database(database) or abort(404)
experiment = db.session.query(
db.Experiment).get(experiment_id) or abort(404)
if not experiment.configuration_scenario: abort(404)
solver_configs = [
sc for sc in experiment.solver_configurations
if sc.solver_binary == experiment.configuration_scenario.solver_binary
]
solver_config_ids = [sc.idSolverConfig for sc in solver_configs]
configurable_parameters = [
p.parameter for p in experiment.configuration_scenario.parameters
if p.configurable and p.parameter.name not in ('instance', 'seed')
]
configurable_parameters_ids = [
p.idParameter for p in configurable_parameters
]
parameter_instances = db.session.query(db.ParameterInstance).options(
joinedload('parameter')).filter(
db.ParameterInstance.SolverConfig_idSolverConfig.in_(
solver_config_ids)).all()
instances_by_id = dict(
(i.idInstance, i) for i in experiment.get_instances(db))
instance_properties = [
p for p in db.session.query(db.Property) if p.is_instance_property()
]
parameter_values = dict()
for pv in parameter_instances:
if pv.Parameters_idParameter not in configurable_parameters_ids:
continue
if pv.SolverConfig_idSolverConfig not in parameter_values:
parameter_values[pv.SolverConfig_idSolverConfig] = dict()
parameter_values[pv.SolverConfig_idSolverConfig][
pv.Parameters_idParameter] = pv.value
results, _, _ = experiment.get_result_matrix(
db,
experiment.solver_configurations,
experiment.get_instances(db),
cost=experiment.defaultCost)
csv_response = StringIO.StringIO()
csv_writer = csv.writer(csv_response)
csv_writer.writerow([p.name for p in configurable_parameters] +
[p.name
for p in instance_properties] + ['par1', 'censored'])
for idInstance in results:
for idSolverConfig in results[idInstance]:
for run in results[idInstance][idSolverConfig]:
csv_writer.writerow(
[parameter_values[idSolverConfig].get(p.idParameter, '') for p in configurable_parameters] + \
[instances_by_id[idInstance].get_property_value(p.idProperty, db) for p in instance_properties] + \
[run.penalized_time1, 1 if run.censored else 0])
csv_response.seek(0)
headers = Headers()
headers.add('Content-Type', 'text/csv')
headers.add('Content-Disposition',
'attachment',
filename=secure_filename(experiment.name) +
"_configuration_runs.csv")
return Response(response=csv_response.read(), headers=headers)
def register(database):
""" User registration """
db = models.get_database(database) or abort(404)
form = forms.RegistrationForm()
errors = []
if form.validate_on_submit():
if db.session.query(db.User).filter_by(email=form.email.data.lower()) \
.count() > 0:
errors.append(
"An account with this email address already exists. Please check your e-mail account for the activation link.")
try:
captcha = map(int, form.captcha.data.split())
if not utils.satisfies(captcha, session['captcha']):
errors.append("Wrong solution to the captcha challenge.")
except:
errors.append("Wrong format of the solution")
if not errors:
user = db.User()
user.lastname = form.lastname.data
user.firstname = form.firstname.data
user.password = password_hash(form.password.data)
user.email = form.email.data.lower() # store email in lower case for easier password reset etc
user.postal_address = form.address.data
user.affiliation = form.affiliation.data
user.verified = False
user.accepted_terms = form.accepted_terms.data
user.affiliation_type = form.affiliation_type.data
user.country = form.country.data
hash = hashlib.sha256()
hash.update(config.SECRET_KEY)
hash.update(user.email)
hash.update(str(datetime.datetime.now()))
hash.update(user.password)
user.activation_hash = hash.hexdigest()
db.session.add(user)
try:
db.session.commit()
except Exception:
db.session.rollback()
errors.append('Error while trying to save the account to the database. Please \
contact an administrator.')
return render('/accounts/register.html', database=database,
db=db, errors=errors, form=form)
session.pop('captcha', None)
msg = Message("[" + db.label + "] Account activation",
recipients=[user.email])
msg.body = "Dear " + user.firstname + " " + user.lastname + ",\n\n" + \
"Please use the following link to activate your account:\n" + \
request.url_root[:-1] + url_for('accounts.activate', database=database,
activation_hash=user.activation_hash)
mail.send(msg)
flash("Account created successfully. An e-mail has been sent to your account with an activation link.")
return redirect(url_for('frontend.experiments_index',
database=database))
# Save captcha to the session. The user will have to provide a solution for
# the same captcha that was given to him.
random.seed()
f = utils.random_formula(2, 3)
while not utils.SAT(f):
f = utils.random_formula(2, 3)
session['captcha'] = f
return render('/accounts/register.html', database=database, db=db,
errors=errors, form=form)
def list_users(database):
db = models.get_database(database) or abort(404)
return render('/accounts/list_users.html', db=db, database=database, users=db.session.query(db.User).all())