def populate(self, db_session):
from datetime import date
from occams_datastore import models as datastore
from occams_studies import models as studies
drsc = studies.Study(name=u'drsc',
title=u'DRSC',
short_title=u'dr',
code=u'drs',
consent_date=date.today(),
is_randomized=False)
schema1 = datastore.Schema(
name=u'demographics',
title=u'demographics',
publish_date=u'2015-01-01',
attributes={
'question':
datastore.Attribute(name=u'question',
title=u'question',
type=u'choice',
order=0,
choices={
u'0':
datastore.Choice(name=u'0',
title=u'always',
order=0),
u'1':
datastore.Choice(name=u'1',
title=u'never',
order=1)
})
})
schema2 = datastore.Schema(
name=u'ucsd_demographics',
title=u'ucsd_demographics',
publish_date=u'2015-01-01',
attributes={
'ucsd_question':
datastore.Attribute(name=u'ucsd_question',
title=u'ucsd_question',
type=u'choice',
order=0,
choices={
u'0':
datastore.Choice(name=u'0',
title=u'always',
order=0),
u'1':
datastore.Choice(name=u'1',
title=u'never',
order=1)
})
})
drsc.schemata.add(schema1)
drsc.schemata.add(schema2)
db_session.add(drsc)
db_session.flush()
def test_attribute_unique_case_insensitive(db_session):
"""
It should enforce case-insensitive attributes
"""
from datetime import date
import sqlalchemy.exc
from occams_datastore import models
schema = models.Schema(name='Foo',
title=u'Foo',
publish_date=date(2014, 3, 31))
schema.attributes['MyAttr'] = models.Attribute(name=u'MyAttr',
title=u'My Attribute',
type=u'string',
order=0)
db_session.add(schema)
db_session.flush()
schema.attributes['myattr'] = models.Attribute(name=u'myattr',
title=u'My Attribute 2',
type=u'string',
order=1)
with pytest.raises(sqlalchemy.exc.IntegrityError):
db_session.flush()
def test_validator_max_constraint(db_session):
"""
It should validate string/number value min/max
"""
from datetime import date
from occams_datastore import models
from occams_datastore.exc import ConstraintError
schema = models.Schema(name=u'Foo', title=u'',
publish_date=date(2000, 1, 1))
s1 = models.Attribute(
schema=schema, name='s1', title=u'Section 1', type='section', order=0)
models.Attribute(
schema=schema,
parent_attribute=s1,
name=u'test',
title=u'',
type=u'string',
is_required=False,
value_max=3,
order=0)
db_session.add(schema)
db_session.flush()
entity = models.Entity(schema=schema)
db_session.add(entity)
entity['test'] = None
with pytest.raises(ConstraintError):
entity['test'] = u'foobar'
entity['test'] = u'foo'
db_session.flush()
assert 'foo' == entity['test']
def populate(self, db_session):
from datetime import date
from occams_datastore import models as datastore
from occams_studies import models as studies
drsc = studies.Study(name=u'drsc',
title=u'DRSC',
short_title=u'dr',
code=u'drs',
consent_date=date.today(),
is_randomized=False)
schema1 = datastore.Schema(name=u'demographics',
title=u'demographics',
publish_date=u'2015-01-01',
attributes={
'myfield':
datastore.Attribute(name=u'myfield',
title=u'My Field',
type=u'string',
order=0),
'question':
datastore.Attribute(name=u'question',
title=u'question',
type=u'string',
order=0)
})
drsc.schemata.add(schema1)
db_session.add(drsc)
db_session.flush()
def test_from_section_to_schema(self, req, db_session, check_csrf_token):
"""
It should be able to move a field from a section to the root
"""
from occams_datastore import models as datastore
schema = datastore.Schema(
name='testform',
title=u'Test Form',
attributes={
'section1':
datastore.Attribute(name='section1',
title=u'Section 1',
type='section',
attributes={
'myvar':
datastore.Attribute(name='myvar',
title=u'My Var',
type='string',
order=1)
},
order=0)
})
db_session.add(schema)
db_session.flush()
req.json_body = {'target': None, 'index': 1}
self._call_fut(schema.attributes['myvar'], req)
assert sorted([(None, 'section1', 0), (None, 'myvar', 1)]) == \
sorted(self._comparable(schema))
def check_value_max_constraint(db_session, type_, limit, below, equal, over):
"""
It should validate against maximum constraints
"""
from datetime import date
from occams_datastore import models
from occams_datastore.exc import ConstraintError
schema = models.Schema(name=u'Foo', title=u'',
publish_date=date(2000, 1, 1))
s1 = models.Attribute(
schema=schema, name='s1', title=u'Section 1', type='section', order=0)
entity = models.Entity(schema=schema)
db_session.add(entity)
db_session.flush()
models.Attribute(
schema=schema, parent_attribute=s1,
name=type_, title=u'', type=type_, is_required=False,
value_max=limit, order=0)
entity[type_] = None
entity[type_] = below
entity[type_] = equal
with pytest.raises(ConstraintError):
entity[type_] = over
models.Attribute(
schema=schema, parent_attribute=s1,
name=u'boolean', title=u'', type=u'boolean', value_max=10, order=1)
with pytest.raises(NotImplementedError):
entity['boolean'] = True
def test_entity_force_date(db_session):
"""
It should maintain a date object for date types.
(Sometimes applications will blindly assign datetimes...)
"""
from datetime import date, datetime
from occams_datastore import models
schema = models.Schema(name=u'Foo', title=u'',
publish_date=date(2000, 1, 1))
s1 = models.Attribute(
schema=schema, name='s1', title=u'Section 1', type='section', order=0)
entity = models.Entity(schema=schema)
# Do simple values
simpleName = 'choicesimple'
schema.attributes[simpleName] = models.Attribute(
schema=schema,
parent_attribute=s1,
title=u'', type='date', is_required=False, order=1)
now = datetime.now()
today = now.date()
entity[simpleName] = now
db_session.flush()
assert isinstance(entity[simpleName], date)
assert today == entity[simpleName]
def test_schema_has_private(db_session):
"""
It should be able to determine if a schema has private attributes
"""
from datetime import date
from occams_datastore import models
schema = models.Schema(name='Foo',
title=u'Foo',
publish_date=date(2014, 3, 31),
attributes={
'not_private':
models.Attribute(name='not_private',
title=u'',
type='string',
is_private=False,
order=0)
})
db_session.add(schema)
db_session.flush()
assert not schema.has_private
schema.attributes['is_private'] = models.Attribute(name='is_private',
title=u'',
type='string',
is_private=True,
order=1)
assert schema.has_private
def test_copy_schema_basic(db_session):
"""
It should let the user copy schemata
"""
from copy import deepcopy
from occams_datastore import models
schema = models.Schema(name='Foo',
title=u'Foo',
attributes={
'section1':
models.Attribute(
name=u'section1',
title=u'Section 1',
type='section',
order=0,
attributes={
'foo':
models.Attribute(
name='foo',
title=u'Enter Foo',
type='choice',
order=1,
choices={
'001':
models.Choice(name='001',
title=u'Foo',
order=0),
'002':
models.Choice(name='002',
title=u'Bar',
order=1),
'003':
models.Choice(name='003',
title=u'Baz',
order=2)
},
)
})
})
db_session.add(schema)
db_session.flush()
schema_copy = deepcopy(schema)
db_session.add(schema_copy)
db_session.flush()
# The ones that matter for checksums
assert schema.name == schema_copy.name
attribute = schema.attributes['foo']
for prop in ('name', 'title', 'description', 'type', 'is_collection',
'is_required'):
attribute_copy = schema_copy.attributes['foo']
assert getattr(attribute, prop) == getattr(attribute_copy, prop)
for choice in schema.attributes['foo'].choices.values():
choice_copy = schema_copy.attributes['foo'].choices[choice.name]
for prop in ('name', 'title', 'order'):
assert getattr(choice, prop) == getattr(choice_copy, prop)
def test_entity_choices(db_session):
"""
It should properly handle choices
"""
from datetime import date
from occams_datastore import models
schema = models.Schema(name=u'Foo', title=u'',
publish_date=date(2000, 1, 1))
s1 = models.Attribute(
schema=schema, name='s1', title=u'Section 1', type='section', order=0)
entity = models.Entity(schema=schema)
db_session.add(entity)
db_session.flush()
# Do simple values
simpleName = 'choicesimple'
schema.attributes[simpleName] = models.Attribute(
schema=schema,
parent_attribute=s1,
name=simpleName,
title=u'', type='choice', is_required=False, order=1,
choices={
'001': models.Choice(name=u'001', title=u'Foo', order=1),
'002': models.Choice(name=u'002', title=u'Bar', order=2),
'003': models.Choice(name=u'003', title=u'Baz', order=3),
'004': models.Choice(name=u'004', title=u'Caz', order=4),
'005': models.Choice(name=u'005', title=u'Jaz', order=5),
})
entity[simpleName] = None
db_session.flush()
assert entity[simpleName] is None
entity[simpleName] = u'002'
db_session.flush()
assert u'002' == entity[simpleName]
# Now try collections
collectionName = 'choicecollection'
schema.attributes[collectionName] = models.Attribute(
schema=schema,
parent_attribute=s1,
name=collectionName,
title=u'', type='choice', is_collection=True, order=2,
choices={
'001': models.Choice(name=u'001', title=u'Foo', order=1),
'002': models.Choice(name=u'002', title=u'Bar', order=2),
'003': models.Choice(name=u'003', title=u'Baz', order=3),
'004': models.Choice(name=u'004', title=u'Caz', order=4),
'005': models.Choice(name=u'005', title=u'Jaz', order=5)})
entity[collectionName] = [u'001', u'002', u'005']
db_session.flush()
assert sorted([u'001', u'002', u'005']) == \
sorted(entity['choicecollection'])
def test_get_attributes_value_value(self):
"""Should return the value to be matched."""
from datetime import date
from occams_datastore import models as datastore
from occams_studies import models as studies
from occams_imports import models
study = studies.Study(name=u'UCSD',
title=u'UCSD',
short_title=u'UCSD',
code=u'001',
consent_date=date(2015, 01, 01))
schema = datastore.Schema(name=u'PatientRegistrationAndDemographics',
title=u'PatientRegistrationAndDemographics',
publish_date=date(2013, 02, 25),
attributes={
'birthyear':
datastore.Attribute(name=u'birthyear',
title=u'birthyear',
type=u'number',
order=0)
})
record = models.SiteData(schema=schema, study=study, data=SITEDATA)
source_variable = u'birthyear'
choices_mapping = []
target_value = self._call_fut(choices_mapping, record, source_variable,
schema)
assert target_value == '35'
def test_get_errors_empty(self, db_session):
"""Should return no errors."""
from datetime import date
from occams_datastore import models as datastore
target_schema = datastore.Schema(name=u'Demographics',
title=u'Demographics',
publish_date=date(2013, 02, 25),
attributes={
'yob':
datastore.Attribute(
name=u'yob',
title=u'yob',
type=u'number',
order=0)
})
db_session.add(target_schema)
db_session.flush()
target_variable = u'yob'
target_value = u'25'
errors = self._call_fut(db_session, target_schema, target_variable,
target_value)
assert errors == []
def test_choice_defaults(db_session):
"""
It should set choice defaults
"""
from occams_datastore import models
schema = models.Schema(name=u'Foo', title=u'Foo')
attribute = models.Attribute(schema=schema,
name=u'foo',
title=u'Enter Foo',
type=u'choice',
order=0)
choice1 = models.Choice(attribute=attribute,
name='001',
title=u'Foo',
order=0)
choice2 = models.Choice(attribute=attribute,
name='002',
title=u'Bar',
order=1)
choice3 = models.Choice(attribute=attribute,
name='003',
title=u'Baz',
order=2)
db_session.add_all([schema, attribute, choice1, choice2, choice3])
db_session.flush()
count = db_session.query(models.Choice).count()
assert count, 3 == 'Did not find any choices'
def test_list_not_include_private(self, db_session):
"""
It should not include private data if specified.
Note this is not the same as de-identification)
"""
from datetime import date
from occams_datastore import models as datastore
from occams_studies.exports.schema import SchemaPlan
schema = datastore.Schema(name=u'contact',
title=u'Contact Details',
publish_date=date.today(),
attributes={
'foo':
datastore.Attribute(name='foo',
title=u'',
type='string',
order=0,
is_private=True)
})
db_session.add_all([schema])
db_session.flush()
plans = SchemaPlan.list_all(db_session, include_private=True)
assert len(plans) == 1
plans = SchemaPlan.list_all(db_session, include_private=False)
assert len(plans) == 0
def test_file(self, req, db_session):
"""
It should return the json rows for the codebook fragment
"""
from datetime import date
from webob.multidict import MultiDict
from occams_datastore import models as datastore
from occams_studies import models
from occams_studies.exports.schema import SchemaPlan
db_session.add(
datastore.Schema(name=u'aform',
title=u'',
publish_date=date.today(),
attributes={
u'myfield':
datastore.Attribute(name=u'myfield',
title=u'',
type=u'string',
order=0)
}))
db_session.flush()
req.GET = MultiDict([('file', 'aform')])
req.registry.settings['studies.export.plans'] = [SchemaPlan.list_all]
res = self._call_fut(models.ExportFactory(req), req)
assert res is not None
def test_file_is_deleted(self, db_session):
"""
Test file is deleted on the system after a non-FieldStorage
object is passed to apply_data
"""
import os
from datetime import date
from occams_datastore import models as datastore
from mock import Mock
schema = datastore.Schema(
name=u'test', title=u'', publish_date=date.today(),
attributes={
'q1': datastore.Attribute(
name=u'q1',
title=u'',
type='blob',
order=0
)
})
entity = datastore.Entity(schema=schema)
formdata = {'q1': u''}
with open(os.path.join(self.tmpdir, 'test.txt'), 'w'):
fullpath = os.path.join(self.tmpdir, 'test.txt')
entity['q1'] = Mock(path=fullpath)
self._call_fut(db_session, entity, formdata, self.tmpdir)
assert not os.path.exists(fullpath)
def test_add_duplicate_variable_name(self, req, db_session,
check_csrf_token):
"""
It should make sure the variable name is not repeated
"""
from pyramid.httpexceptions import HTTPBadRequest
from occams_datastore import models as datastore
schema = datastore.Schema(name='testform',
title=u'Test Form',
attributes={
'myvar':
datastore.Attribute(name='myvar',
title=u'My Var',
type='string',
order=0)
})
db_session.add(schema)
db_session.flush()
req.json_body = {'name': 'myvar'}
with pytest.raises(HTTPBadRequest) as excinfo:
self._call_fut(schema['fields'], req)
assert check_csrf_token.called
assert 'name already exists' in \
excinfo.value.json['errors']['name'].lower()
def test_add_section_into_section(self, req, db_session, check_csrf_token):
"""
It should not allow adding a new section into a another section
"""
from pyramid.httpexceptions import HTTPBadRequest
from occams_datastore import models as datastore
schema = datastore.Schema(name='testform',
title=u'Test Form',
attributes={
'section1':
datastore.Attribute(name='section1',
title=u'Section 1',
type='section',
order=0)
})
db_session.add(schema)
db_session.flush()
req.json_body = {
'target': 'section1',
'name': 'section2',
'title': u'Section 2',
'type': 'section'
}
with pytest.raises(HTTPBadRequest) as excinfo:
self._call_fut(schema['fields'], req)
assert 'nested sections are not supported' in \
excinfo.value.json['errors']['target'].lower()
def test_choice_multi(self):
from occams_datastore import models as datastore
from occams_forms.renderers import make_field
attribute = datastore.Attribute(
name=u'f', title=u'F', type='choice', is_collection=True)
field = make_field(attribute)
assert field.field_class is wtforms.SelectMultipleField
def test_integer(self):
from occams_datastore import models as datastore
from occams_forms.renderers import make_field
attribute = datastore.Attribute(
name=u'f', title=u'F', type='number', decimal_places=0)
field = make_field(attribute)
assert field.field_class is wtforms.fields.html5.IntegerField
def edit_json(context, request):
"""
Add/Edit form for fields.
"""
check_csrf_token(request)
db_session = request.db_session
form = FieldFormFactory(context, request).from_json(request.json_body)
if not form.validate():
raise HTTPBadRequest(json={'errors': wtferrors(form)})
is_new = isinstance(context, models.AttributeFactory)
if not is_new:
attribute = context
else:
# Add the attribute and temporarily set to large display order
attribute = datastore.Attribute(schema=context.__parent__, order=-1)
db_session.add(attribute)
attribute.apply(form.data)
if is_new:
# now we can move the attribute
move_json(attribute, request)
db_session.flush()
return view_json(attribute, request)
def validate_populate_imports(request, records):
"""
Return list of errors, imports and forms
:request: request obj
:records: a list of csv row data
:return: errors, imports, and forms for template rendering
"""
errors, imports, forms = ([], [], [])
for record in records:
schema_dict = {
'name': record['schema_name'],
'title': record['schema_title'],
'publish_date': record['publish_date'].strftime('%Y-%m-%d')
}
FormForm = FormFormFactory(context=None, request=request)
form_form = FormForm.from_json(schema_dict)
if not form_form.validate():
schema_error = {}
schema_error['errors'] = wtferrors(form_form)
schema_error['schema_name'] = schema_dict['name']
schema_error['schema_title'] = schema_dict['title']
schema_error['name'] = 'N/A'
schema_error['title'] = 'N/A'
errors.append(schema_error)
else:
schema = datastore.Schema.from_json(schema_dict)
if schema.to_json() not in forms:
forms.append(schema.to_json())
choices = parse.get_choices(record['choices'])
# below needed because we are calling from_json on record
record['choices'] = choices
FieldForm = FieldFormFactory(context=schema, request=request)
form = FieldForm.from_json(record)
if not form.validate():
output = log_errors(wtferrors(form), record)
errors.append(output)
else:
imports.append((datastore.Attribute(
name=record['name'],
title=record['title'],
description=record['description'],
is_required=record['is_required'],
is_collection=record['is_collection'],
is_private=record['is_private'],
type=record['type'],
order=record['order'],
choices=choices
), schema))
return errors, imports, forms
def test_one_record_exists_db_after_update(self, db_session):
"""
Test if one updated record exists in value_blob tbl
after FieldStorage object passed to apply_data
"""
import os
import cgi
from datetime import date
from occams_datastore import models as datastore
from mock import Mock
schema = datastore.Schema(
name=u'test', title=u'', publish_date=date.today(),
attributes={
'q1': datastore.Attribute(
name=u'q1',
title=u'',
type='blob',
order=0
)
})
entity = datastore.Entity(schema=schema)
form = cgi.FieldStorage()
form.filename = u'test.txt'
form.file = form.make_file()
form.file.write(u'test_content')
form.file.seek(0)
formdata = {'q1': form}
with open(os.path.join(self.tmpdir, 'test.txt'), 'w'):
fullpath = os.path.join(self.tmpdir, 'test.txt')
entity['q1'] = Mock(path=fullpath)
self._call_fut(db_session, entity, formdata, self.tmpdir)
blob = db_session.query(datastore.ValueBlob).one()
entity_id = blob.entity.id
form_update = cgi.FieldStorage()
form_update.filename = u'test2.txt'
form_update.file = form.make_file()
form_update.file.write(u'test_content')
form_update.file.seek(0)
formdata2 = {'q1': form_update}
self._call_fut(db_session, entity, formdata2, self.tmpdir)
blob = db_session.query(datastore.ValueBlob).filter_by(
file_name=u'test2.txt').first()
entity_id_after_update = blob.entity_id
assert entity_id == entity_id_after_update
def test_valid_upload(self, req, db_session, check_csrf_token):
"""
It should be able to upload a perfectly valid CSV
"""
import tempfile
import csv
from datetime import date
from occams_datastore import models as datastore
from occams_studies import models
schema = datastore.Schema(name='rand',
title=u'Rand',
publish_date=date.today(),
attributes={
'criteria':
datastore.Attribute(name='criteria',
title=u'Criteria',
type='string',
order=0)
})
study = models.Study(name=u'somestudy',
title=u'Some Study',
short_title=u'sstudy',
code=u'000',
is_randomized=True,
randomization_schema=schema,
consent_date=date.today())
db_session.add_all([study])
db_session.flush()
class DummyUpload:
pass
with tempfile.NamedTemporaryFile(prefix='nose-', suffix='.exe') as fp:
upload = DummyUpload()
upload.file = fp
upload.filename = fp.name
# forget the schema keys
writer = csv.writer(fp)
writer.writerow(
[u'ARM', u'STRATA', u'BLOCKID', u'RANDID',
u'CRITERIA']) # noqa
writer.writerow(
[u'UCSD', u'hints', u'1234567', u'987654',
u'is smart']) # noqa
fp.flush()
req.POST = {'upload': upload}
self._call_fut(study, req)
stratum = db_session.query(models.Stratum).one()
entity = db_session.query(datastore.Entity).one()
assert stratum.arm.name == 'UCSD'
assert entity in stratum.entities
assert entity['criteria'] == 'is smart'
def test_decimal_precision(self):
from occams_datastore import models as datastore
from occams_forms.renderers import make_field
attribute = datastore.Attribute(
name=u'f', title=u'F', type='number', decimal_places=1)
field = make_field(attribute)
assert field.field_class is wtforms.fields.html5.DecimalField
assert field.kwargs['places'] == 1
assert field.kwargs['rounding'] == u'ROUND_UP'
def test_daterange_datetime(self):
from occams_datastore import models as datastore
from occams_forms.renderers import make_field
import wtforms
from wtforms_components import DateRange
attribute = datastore.Attribute(
name=u'daterange_test', title=u'daterange_test', type='datetime')
field = make_field(attribute)
field = field.bind(wtforms.Form(), attribute.name)
assert any(isinstance(v, DateRange) for v in field.validators)
def test_string_min_max_same_value(self):
from occams_datastore import models as datastore
from occams_forms.renderers import make_field
import wtforms
from wtforms.validators import Length
attribute = datastore.Attribute(
name=u'string_test', title=u'string_test', type='string',
value_min=3, value_max=3)
field = make_field(attribute)
field = field.bind(wtforms.Form(), attribute.name)
assert any(isinstance(v, Length) for v in field.validators)
def test_number_min_max_same_number(self):
from occams_datastore import models as datastore
from occams_forms.renderers import make_field
import wtforms
from wtforms.validators import NumberRange
attribute = datastore.Attribute(
name=u'number_test', title=u'number_test', type='number',
value_min=3, value_max=3)
field = make_field(attribute)
field = field.bind(wtforms.Form(), attribute.name)
assert any(isinstance(v, NumberRange) for v in field.validators)
def test_multiple_choice_min_max(self):
from occams_datastore import models as datastore
from occams_forms.renderers import make_field
import wtforms
from wtforms.validators import Length
attribute = datastore.Attribute(
name=u'choice', title=u'choice_test', type='choice',
is_collection=True,
value_min=1, value_max=12)
field = make_field(attribute)
field = field.bind(wtforms.Form(), attribute.name)
assert any(isinstance(v, Length) for v in field.validators)
def test_choice_constraint(db_session):
"""
It should validate against choice constraints
"""
from datetime import date
from occams_datastore import models
from occams_datastore.exc import ConstraintError
schema = models.Schema(name=u'Foo', title=u'',
publish_date=date(2000, 1, 1))
s1 = models.Attribute(
schema=schema, name='s1', title=u'Section 1', type='section', order=0)
models.Attribute(
schema=schema, parent_attribute=s1,
name=u'test', title=u'', type=u'choice', is_required=False, order=0,
choices={
'001': models.Choice(name=u'001', title=u'Foo', order=0),
'002': models.Choice(name=u'002', title=u'Bar', order=1),
'003': models.Choice(name=u'003', title=u'Baz', order=2)})
db_session.add(schema)
db_session.flush()
entity = models.Entity(schema=schema)
db_session.add(entity)
entity['test'] = None
entity['test'] = u'002'
db_session.flush()
entry = (
db_session.query(models.ValueChoice)
.filter(models.ValueChoice.value.has(name=u'002'))
.one())
assert entry.value.name == '002'
# Should not be able to set it to something outside of the specified
# choice constraints
with pytest.raises(ConstraintError):
entity['test'] = u'999'