class DBTester(object):
"""The :class:`~ess_test.conftest.DBTester` provides functionality for interacting with the
database..
"""
def __init__(self):
self._dbsession = DBSession()
def get_model(self, name, query_params = None):
"""Retrieve a single instance of the model ``name``, filtered by the ``query``.
:param name: The name of the model to get the instance for
:type name: ``unicode``
:param query: The query to use for selecting the instance as a tuple of ``(field, comparator, value)``
:type query: ``tuple``
:return: The result of the query
"""
cls = getattr(models, name)
query = self._dbsession.query(cls)
if query_params:
if query_params[1] == '==':
query = query.filter(getattr(cls, query_params[0]) == query_params[2])
return query.first()
def create_model(self, name, params):
"""Create a new instance of the model ``name`` with the given ``params``.
:param name: The name of the model to create the instance of
:type name: ``unicode``
:param params: The initial parameters to use for creating the instance
:type params: ``dict``
:return: The new instance
"""
cls = getattr(models, name)
loaded = list(self._dbsession.identity_map.values())
with transaction.manager:
model = cls(**params)
self._dbsession.add(model)
self._dbsession.add(model)
self._dbsession.add_all(loaded)
return model
def update(self, obj, **kwargs):
"""Update the given ``obj`` with the key/value parameters.
:param obj: The obj to update
:type obj: :class:`pywebtools.sqlalchemy.Base`
"""
loaded = list(self._dbsession.identity_map.values())
with transaction.manager:
self._dbsession.add(obj)
for key, value in kwargs.items():
if isinstance(value, Base):
self._dbsession.add(value)
setattr(obj, key, value)
self._dbsession.add_all(loaded)
def flush(self):
"""Flush the session."""
self._dbsession.flush()
def action(request):
"""Handles the ``/users/action`` URL, applying the given action to the
list of selected users. Requires that the current
:class:`~wte.models.User` has the "admin.users.view"
:class:`~wte.models.Permission`.
"""
dbsession = DBSession()
try:
query_params = []
for param in ['q', 'status', 'start']:
if param in request.params and request.params[param]:
query_params.append((param, request.params[param]))
params = ActionSchema().to_python(request.params,
State(request=request))
if params['action'] != 'delete' or params['confirm']:
with transaction.manager:
for user in dbsession.query(User).filter(
User.id.in_(params['user_id'])):
if params['action'] == 'validate':
if user.status == 'unconfirmed' and user.allow(
'edit', request.current_user):
user.status = 'active'
elif params['action'] == 'delete':
if user.allow('delete', request.current_user):
dbsession.delete(user)
elif params['action'] == 'password':
if user.status == 'active' and user.allow(
'edit', request.current_user):
token = TimeToken(
user.id, 'reset_password',
datetime.now() + timedelta(seconds=1200))
dbsession.add(token)
dbsession.flush()
if 'user.password_reset' in active_callbacks:
active_callbacks['user.password_reset'](
request, user, token)
raise HTTPSeeOther(request.route_url('users', _query=query_params))
else:
return {
'params':
params,
'users':
dbsession.query(User).filter(User.id.in_(params['user_id'])),
'query_params':
query_params,
'crumbs':
create_user_crumbs(request, [{
'title': 'Confirm',
'url': request.current_route_url()
}])
}
except Invalid as e:
print(e)
request.session.flash(
'Please select the action you wish to apply and the users to apply it to',
queue='error')
raise HTTPSeeOther(request.route_url('users', _query=query_params))
def import_experiment(request):
if request.method == 'POST':
try:
params = ImportExperimentSchema().to_python(
request.params, State(request=request))
dbsession = DBSession()
with transaction.manager:
experiment, errors = ExperimentIOSchema(include_schemas=[s for s in all_io_schemas
if s != ExperimentIOSchema]).\
loads(params['source'].file.read().decode('utf-8'))
if errors:
raise formencode.Invalid(
'. '.join([
'%s: %s' % (e['source']['pointer'], e['detail'])
for e in errors['errors']
]), None, None)
for page in experiment.pages:
replace_questions(page, dbsession)
experiment.owner = request.current_user
experiment.external_id = uuid.uuid1().hex,
experiment.status = 'develop'
dbsession.add(experiment)
dbsession.flush()
for latin_square in experiment.latin_squares:
fix_latinsquare(latin_square, experiment.data_sets)
for page in experiment.pages:
for transition in page.next:
fix_transition(transition, experiment.pages)
dbsession.add(experiment)
raise HTTPFound(
request.route_url('experiment.view', eid=experiment.id))
except formencode.Invalid as e:
return {
'crumbs': [{
'title': 'Experiments',
'url': request.route_url('dashboard')
}, {
'title': 'Import',
'url': request.route_url('experiment.import')
}],
'errors': {
'source': str(e)
},
'values':
request.params
}
return {
'crumbs': [{
'title': 'Experiments',
'url': request.route_url('dashboard')
}, {
'title': 'Import',
'url': request.route_url('experiment.import')
}]
}
def run_timed_tasks(args):
"""Runs all timed tasks where the timestamp is in the past and the status is "ready".
All :class:`~wte.models.TimedTask` that are to be run are given a unique "run-{random-number}"
``status`` to uniquely identify them for this run. Individual task runners are then
responsible for setting that status to "completed" after the task completes successfully
or to "failed" if it failed.
All task runners are run in independent :class:`threading.Thread`\ s. After all
:class:`~threading.Thread` complete, any :class:`~wte.models.TimedTask` that still have
the unique "run-{random-number}" status are automatically set to the "failed" status.
"""
settings = get_appsettings(args.configuration)
setup_logging(args.configuration)
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.bind = engine
dbsession = DBSession()
tasks = dbsession.query(TimedTask).filter(
and_(TimedTask.timestamp <= func.now(), TimedTask.status == 'ready'))
rnd = randint(0, 1000000)
with transaction.manager:
tasks.update({TimedTask.status: 'running-%i' % (rnd)},
synchronize_session=False)
tasks = dbsession.query(TimedTask).filter(
TimedTask.status == 'running-%i' % (rnd))
task_count = tasks.count()
if task_count > 0:
logging.getLogger('wte').info('Running %i tasks' % (task_count))
threads = []
for task in tasks:
if task.name == 'change_status':
threads.append(
Thread(None, target=run_change_status, args=(task.id, )))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
dbsession.flush()
failed_count = tasks.count()
if failed_count > 0:
logging.getLogger('wte').error('%i tasks failed' % (failed_count))
else:
logging.getLogger('wte').info('All tasks completed')
with transaction.manager:
tasks.update({TimedTask.status: 'failed'})
def actions_duplicate(request):
dbsession = DBSession()
experiment = dbsession.query(Experiment).filter(
Experiment.id == request.matchdict['eid']).first()
if experiment:
if request.method == 'POST':
try:
params = DuplicateSchema().to_python(request.params,
State(request=request))
data = ExperimentIOSchema(include_schemas=[s for s in all_io_schemas if s != ExperimentIOSchema]).\
dump(experiment).data
data = deepcopy(data)
with transaction.manager:
dbsession.add(experiment)
new_experiment, errors = ExperimentIOSchema(include_schemas=[s for s in all_io_schemas
if s != ExperimentIOSchema]).\
load(data)
if errors:
raise formencode.Invalid(
'. '.join([
'%s: %s' %
(e['source']['pointer'], e['detail'])
for e in errors['errors']
]), None, None)
for page in new_experiment.pages:
replace_questions(page, dbsession)
new_experiment.title = params['title']
new_experiment.owner = request.current_user
new_experiment.external_id = uuid.uuid1().hex,
new_experiment.status = 'develop'
dbsession.add(new_experiment)
dbsession.flush()
for latin_square in new_experiment.latin_squares:
fix_latin_square(latin_square,
new_experiment.data_sets)
for page in new_experiment.pages:
for transition in page.next:
fix_transition(transition, new_experiment.pages)
dbsession.add(new_experiment)
raise HTTPFound(
request.route_url('experiment.view',
eid=new_experiment.id))
except formencode.Invalid as e:
return {
'experiment':
experiment,
'crumbs': [{
'title': 'Experiments',
'url': request.route_url('dashboard')
}, {
'title':
experiment.title,
'url':
request.route_url('experiment.view', eid=experiment.id)
}, {
'title':
'Actions',
'url':
request.route_url('experiment.actions',
eid=experiment.id)
}, {
'title':
'Duplicate',
'url':
request.route_url('experiment.actions.duplicate',
eid=experiment.id)
}],
'errors':
e.error_dict if e.error_dict else {
'title': str(e)
},
'values':
request.params
}
return {
'experiment':
experiment,
'crumbs': [{
'title': 'Experiments',
'url': request.route_url('dashboard')
}, {
'title':
experiment.title,
'url':
request.route_url('experiment.view', eid=experiment.id)
}, {
'title':
'Actions',
'url':
request.route_url('experiment.actions', eid=experiment.id)
}, {
'title':
'Duplicate',
'url':
request.route_url('experiment.actions.duplicate',
eid=experiment.id)
}]
}
else:
raise HTTPNotFound()
def latinsquare_edit(request):
dbsession = DBSession()
experiment = dbsession.query(Experiment).filter(
Experiment.id == request.matchdict['eid']).first()
data_set = dbsession.query(DataSet).filter(
and_(DataSet.id == request.matchdict['did'],
DataSet.experiment_id == request.matchdict['eid'],
DataSet.type == 'latinsquare')).first()
if experiment and data_set:
if request.method == 'POST':
try:
params = LatinSquareEditSchema().to_python(
request.params,
State(request=request,
dbsession=dbsession,
experiment=experiment,
data_set=data_set))
count = latinsquare_estimate_count(dbsession,
params['source_a'],
params['mode_a'],
params['source_b'],
params['mode_b'])
if count > 10000:
raise formencode.Invalid(
'',
None,
None,
error_dict={
'source_a':
'These settings generate too many combinations (%s).'
% count, # noqa: E501
'source_b':
'These settings generate too many combinations (%s).'
% count
}) # noqa: E501
with transaction.manager:
dbsession.add(data_set)
source_a = dbsession.query(DataSet).filter(
and_(DataSet.id == params['source_a'],
DataSet.type == 'dataset')).first()
source_b = dbsession.query(DataSet).filter(
and_(DataSet.id == params['source_b'],
DataSet.type == 'dataset')).first()
data_set.name = params['name']
data_set['source_a'] = int(params['source_a'])
data_set['mode_a'] = params['mode_a']
data_set['source_b'] = int(params['source_b'])
data_set['mode_b'] = params['mode_b']
data_set[
'columns'] = source_a['columns'] + source_b['columns']
data_set.items.clear()
combinations = []
if params['mode_a'] == 'within' and params[
'mode_b'] == 'within':
iterator = itertools.permutations(
itertools.product(source_a.items, source_b.items))
elif params['mode_a'] == 'between' and params[
'mode_b'] == 'within':
iterator = []
for item_a in source_a.items:
iterator.extend(
itertools.permutations(
itertools.product([item_a],
source_b.items)))
elif params['mode_a'] == 'within' and params[
'mode_b'] == 'between':
iterator = []
for item_b in source_b.items:
iterator.extend(
itertools.permutations(
itertools.product(source_a.items,
[item_b])))
elif params['mode_a'] == 'between' and params[
'mode_b'] == 'between':
iterator = [[p] for p in itertools.product(
source_a.items, source_b.items)]
order = 0
for data in iterator:
comb = []
for pair in data:
item = DataItem(data_set=data_set, order=order)
values = {}
values.update(pair[0]['values'])
values.update(pair[1]['values'])
item['values'] = values
comb.append(item)
order = order + 1
combinations.append(comb)
dbsession.flush()
data_set['combinations'] = [[di.id for di in c]
for c in combinations]
dbsession.add(experiment)
dbsession.add(data_set)
raise HTTPFound(
request.route_url('experiment.latinsquare.view',
eid=experiment.id,
did=data_set.id))
except formencode.Invalid as e:
return {
'experiment':
experiment,
'data_set':
data_set,
'crumbs': [{
'title': 'Experiments',
'url': request.route_url('dashboard')
}, {
'title':
experiment.title,
'url':
request.route_url('experiment.view', eid=experiment.id)
}, {
'title':
'Latin Squares',
'url':
request.route_url('experiment.latinsquare',
eid=experiment.id)
}, {
'title':
data_set.name,
'url':
request.route_url('experiment.latinsquare.view',
eid=experiment.id,
did=data_set.id)
}, {
'title':
'Edit',
'url':
request.route_url('experiment.latinsquare.edit',
eid=experiment.id,
did=data_set.id)
}],
'values':
request.params,
'errors':
e.error_dict
}
return {
'experiment':
experiment,
'data_set':
data_set,
'crumbs': [{
'title': 'Experiments',
'url': request.route_url('dashboard')
}, {
'title':
experiment.title,
'url':
request.route_url('experiment.view', eid=experiment.id)
}, {
'title':
'Latin Squares',
'url':
request.route_url('experiment.latinsquare', eid=experiment.id)
}, {
'title':
data_set.name,
'url':
request.route_url('experiment.latinsquare.view',
eid=experiment.id,
did=data_set.id)
}, {
'title':
'Edit',
'url':
request.route_url('experiment.latinsquare.edit',
eid=experiment.id,
did=data_set.id)
}]
}
else:
raise HTTPNotFound()