def setUp(self):
management.call_command('avocado', 'sync', 'export', quiet=True)
first_name_field = DataField.objects.get_by_natural_key('export', 'employee', 'first_name')
first_name_field.description = 'First Name'
last_name_field = DataField.objects.get_by_natural_key('export', 'employee', 'last_name')
last_name_field.description = 'Last Name'
title_field = DataField.objects.get_by_natural_key('export', 'title', 'name')
title_field.description = 'Employee Title'
salary_field = DataField.objects.get_by_natural_key('export', 'title', 'salary')
salary_field.description = 'Salary'
is_manager_field = DataField.objects.get_by_natural_key('export', 'employee', 'is_manager')
is_manager_field.description = 'Is a Manager?'
[x.save() for x in [first_name_field, last_name_field, title_field,
salary_field, is_manager_field]]
employee_concept = DataConcept()
employee_concept.name = 'Employee'
employee_concept.description = 'A Single Employee'
employee_concept.save()
DataConceptField(concept=employee_concept, field=first_name_field, order=1).save()
DataConceptField(concept=employee_concept, field=last_name_field, order=2).save()
DataConceptField(concept=employee_concept, field=is_manager_field, order=3).save()
DataConceptField(concept=employee_concept, field=title_field, order=4).save()
DataConceptField(concept=employee_concept, field=salary_field, order=5).save()
self.concepts = [employee_concept]
self.query = models.Employee.objects.values_list('first_name', 'last_name',
'is_manager', 'title__name', 'title__salary')
def setUp(self):
management.call_command("avocado", "init", "tests", quiet=True)
first_name_field = DataField.objects.get_by_natural_key("tests", "employee", "first_name")
first_name_field.description = "First Name"
last_name_field = DataField.objects.get_by_natural_key("tests", "employee", "last_name")
last_name_field.description = "Last Name"
title_field = DataField.objects.get_by_natural_key("tests", "title", "name")
title_field.description = "Employee Title"
salary_field = DataField.objects.get_by_natural_key("tests", "title", "salary")
salary_field.description = "Salary"
is_manager_field = DataField.objects.get_by_natural_key("tests", "employee", "is_manager")
is_manager_field.description = "Is a Manager?"
[x.save() for x in [first_name_field, last_name_field, title_field, salary_field, is_manager_field]]
employee_concept = DataConcept()
employee_concept.name = "Employee"
employee_concept.description = "A Single Employee"
employee_concept.save()
DataConceptField(concept=employee_concept, field=first_name_field, order=1).save()
DataConceptField(concept=employee_concept, field=last_name_field, order=2).save()
DataConceptField(concept=employee_concept, field=is_manager_field, order=3).save()
DataConceptField(concept=employee_concept, field=title_field, order=4).save()
DataConceptField(concept=employee_concept, field=salary_field, order=5).save()
self.concepts = [employee_concept]
self.query = models.Employee.objects.values_list(
"first_name", "last_name", "is_manager", "title__name", "title__salary"
)
def test_published(self):
concept = DataConcept(published=True)
concept.save()
DataConceptField(concept=concept, field=self.is_manager).save()
DataConceptField(concept=concept, field=self.salary).save()
# Published, not specific to any user
self.assertEqual([x.pk for x in DataConcept.objects.published()], [])
self.is_manager.published = True
self.is_manager.save()
self.salary.published = True
self.salary.save()
# Now published, it will appear
self.assertEqual([x.pk for x in DataConcept.objects.published()], [1])
user1 = User.objects.create_user('user1', 'user1')
# Nothing since user1 cannot view either datafield
self.assertEqual([x.pk for x in DataConcept.objects.published(user1)], [])
assign('avocado.view_datafield', user1, self.is_manager)
# Still nothing since user1 has no permission for salary
self.assertEqual([x.pk for x in DataConcept.objects.published(user1)], [])
assign('avocado.view_datafield', user1, self.salary)
# Now user1 can see the concept
self.assertEqual([x.pk for x in DataConcept.objects.published(user1)], [1])
user2 = User.objects.create_user('user2', 'user2')
# `user2` is not assigned
self.assertEqual([x.pk for x in DataConcept.objects.published(user2)], [])
def test_implicit_apply_distinct(self):
f1 = DataField.objects.get_by_natural_key('tests',
'office',
'location')
f2 = DataField.objects.get_by_natural_key('tests',
'title',
'name')
c = DataConcept()
c.save()
DataConceptField(concept=c, field=f1).save()
DataConceptField(concept=c, field=f2).save()
# Due to the use of distinct, the concept fields appear in the SELECT
# statement at this point. This is not a bug, but a requirement of SQL.
# These columns are stripped downstream by the exporter.
node = parsers.dataview.parse([{
'concept': c.pk,
'sort': 'desc',
'visible': False,
}], tree=Employee)
self.assertEqual(
unicode(node.apply(Employee.objects.distinct()).query)
.replace(' ', ''),
'SELECT DISTINCT "tests_employee"."id", '
'"tests_office"."location", "tests_title"."name" FROM '
'"tests_employee" INNER JOIN "tests_office" ON '
'("tests_employee"."office_id" = "tests_office"."id") LEFT OUTER '
'JOIN "tests_title" ON ("tests_employee"."title_id" = '
'"tests_title"."id") ORDER BY "tests_office"."location" DESC, '
'"tests_title"."name" DESC'
.replace(' ', ''))
def create_dataconcept_single(self, request, queryset):
fields = list(queryset)
max_length = DataConcept._meta.get_field_by_name('name')[0].max_length
name = ', '.join([f.name for f in fields])[:max_length - 5] + '...'
concept = DataConcept(name='"{0}"'.format(name))
concept.save()
for i, datafield in enumerate(queryset):
DataConceptField(concept=concept, field=datafield, order=i).save()
def setUp(self):
management.call_command('avocado', 'init', 'query', quiet=True)
f1 = DataField.objects.get(pk=1)
f2 = DataField.objects.get(pk=2)
c1 = DataConcept()
c1.save()
DataConceptField(concept=c1, field=f1).save()
DataConceptField(concept=c1, field=f2).save()
def setUp(self):
super(ConceptResourceTestCase, self).setUp()
self.name_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'name')
self.salary_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'salary')
self.boss_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'boss')
cat = DataCategory(name='Test', published=True, order=1.0)
cat.save()
c1 = DataConcept(name='Title', published=True, category=cat)
c1.save()
DataConceptField(concept=c1, field=self.name_field, order=1).save()
DataConceptField(concept=c1, field=self.salary_field, order=2).save()
DataConceptField(concept=c1, field=self.boss_field, order=3).save()
# Not published
c2 = DataConcept(name='Salary', category=cat)
c2.save()
DataConceptField(concept=c2, field=self.salary_field, order=1).save()
DataConceptField(concept=c2, field=self.boss_field, order=2).save()
c3 = DataConcept(name='Name', published=True, category=cat)
c3.save()
DataConceptField(concept=c3, field=self.name_field, order=1).save()
self.c1 = c1
self.c2 = c2
self.c3 = c3
def test_dataview_order_by(self):
f = DataField(app_name='lexicon', model_name='month', field_name='id')
f.save()
c = DataConcept()
c.save()
cf = DataConceptField(field=f, concept=c)
cf.save()
v = DataView({'ordering': [c.pk]})
qs = Month.objects.filter(label__startswith='J').values('id')
self.assertEqual(str(v.apply(qs).query), 'SELECT "lexicon_month"."id" FROM "lexicon_month" WHERE "lexicon_month"."label" LIKE J% ESCAPE \'\\\' ORDER BY "lexicon_month"."order" ASC')
def create_dataconcept_single(self, request, queryset):
fields = list(queryset)
# If only a single field is selected with this action, directly map
if len(fields) == 1:
DataConcept.objects.create_from_field(fields[0])
return
max_length = DataConcept._meta.get_field_by_name('name')[0].max_length
name = ', '.join([f.name for f in fields])[:max_length - 5] + '...'
concept = DataConcept(name=u'"{0}"'.format(name))
concept.save()
for i, datafield in enumerate(queryset):
DataConceptField(concept=concept, field=datafield, order=i).save()
def create_dataconcept_single(self, request, queryset):
fields = list(queryset)
# If only a single field is selected with this action, directly map
if len(fields) == 1:
DataConcept.objects.create_from_field(fields[0])
return
max_length = DataConcept._meta.get_field_by_name("name")[0].max_length
name = ", ".join([f.name for f in fields])[: max_length - 5] + "..."
concept = DataConcept(name=u'"{0}"'.format(name))
concept.save()
for i, datafield in enumerate(queryset):
DataConceptField(concept=concept, field=datafield, order=i).save()
def test_field_for_concept(self):
f = DataField.objects.get(model_name='title', field_name='name')
c1 = DataConcept()
c2 = DataConcept()
c1.save()
c2.save()
cf = DataConceptField(concept=c1, field=f)
cf.save()
attrs = {
'concept': c1.pk,
'field': f.pk,
'operator': 'exact',
'value': 'CEO',
'cleaned_value': {'value': 'CEO', 'label': 'CEO'},
'language': 'Name is CEO'
}
self.assertEqual(
parsers.datacontext.validate(deepcopy(attrs), tree=Employee),
attrs)
# Invalid concept
attrs = parsers.datacontext.validate({
'concept': c2.pk,
'field': f.pk,
'operator': 'exact',
'value': 'CEO',
}, tree=Employee)
self.assertFalse(attrs['enabled'])
def setUp(self):
management.call_command('avocado', 'init', 'tests', publish=False,
concepts=False, quiet=True)
f1 = DataField.objects.get_by_natural_key('tests',
'employee',
'first_name')
f2 = DataField.objects.get_by_natural_key('tests',
'employee',
'last_name')
self.c = DataConcept()
self.c.save()
DataConceptField(concept=self.c, field=f1).save()
DataConceptField(concept=self.c, field=f2).save()
def setUp(self):
management.call_command('avocado',
'init',
'tests',
publish=False,
concepts=False,
quiet=True)
f1 = DataField.objects.get(pk=1)
f2 = DataField.objects.get(pk=2)
c1 = DataConcept()
c1.save()
DataConceptField(concept=c1, field=f1).save()
DataConceptField(concept=c1, field=f2).save()
def setUp(self):
management.call_command('avocado', 'init', 'tests', quiet=True)
f1 = DataField.objects.get(pk=1)
f2 = DataField.objects.get(pk=2)
c1 = DataConcept()
c1.save()
DataConceptField(concept=c1, field=f1).save()
DataConceptField(concept=c1, field=f2).save()
self.request = HttpRequest()
self.request.session = SessionStore()
self.request.session.save()
def setUp(self):
management.call_command('avocado', 'init', 'tests', quiet=True)
f1 = DataField.objects.get(pk=1)
f2 = DataField.objects.get(pk=2)
c1 = DataConcept()
c1.save()
DataConceptField(concept=c1, field=f1).save()
DataConceptField(concept=c1, field=f2).save()
self.request = HttpRequest()
self.request.session = SessionStore()
self.request.session.save()
def setUp(self):
super(ConceptFieldResourceTestCase, self).setUp()
self.name_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'name')
self.salary_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'salary')
self.boss_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'boss')
c1 = DataConcept(name='Title', published=True)
c1.save()
DataConceptField(concept=c1, field=self.name_field, order=1).save()
DataConceptField(concept=c1, field=self.salary_field, order=2).save()
DataConceptField(concept=c1, field=self.boss_field, order=3).save()
def test_unique_keys(self):
title_name = DataField.objects.get_by_natural_key('tests', 'title', 'name')
project_name = DataField.objects.get_by_natural_key('tests', 'project', 'name')
concept = DataConcept()
concept.save()
DataConceptField(concept=concept, field=title_name, order=1).save()
DataConceptField(concept=concept, field=project_name, order=2).save()
f = Formatter(concept)
self.assertEqual(OrderedDict([
('title__name', 'one'),
('project__name', 'two'),
]), f(['one', 'two']))
def setUp(self):
management.call_command('avocado', 'init', 'tests', quiet=True)
f1 = DataField.objects.get_by_natural_key('tests', 'employee',
'first_name')
f2 = DataField.objects.get_by_natural_key('tests', 'employee',
'last_name')
self.c = DataConcept()
self.c.save()
DataConceptField(concept=self.c, field=f1).save()
DataConceptField(concept=self.c, field=f2).save()
self.request = HttpRequest()
self.request.session = SessionStore()
self.request.session.save()
def test_dataview_order_by(self):
f = DataField(app_name='tests', model_name='month', field_name='id')
f.save()
c = DataConcept()
c.save()
cf = DataConceptField(field=f, concept=c)
cf.save()
v = DataView({'ordering': [[c.pk, 'asc']]})
qs = Month.objects.filter(label__startswith='J').values('id')
self.assertEqual(
unicode(v.apply(qs).query),
'SELECT "tests_month"."id" FROM "tests_month" WHERE "tests_month"."label" LIKE J% ESCAPE \'\\\' ORDER BY "tests_month"."order" ASC'
)
def test_unique_keys(self):
title_name = DataField.objects.get_by_natural_key(
'tests', 'title', 'name')
project_name = DataField.objects.get_by_natural_key(
'tests', 'project', 'name')
concept = DataConcept()
concept.save()
DataConceptField(concept=concept, field=title_name, order=1).save()
DataConceptField(concept=concept, field=project_name, order=2).save()
f = Formatter(concept=concept)
meta = f.get_meta()
names = [x['name'] for x in meta['header']]
self.assertEqual(names, ['title__name', 'project__name'])
def setUp(self):
management.call_command('avocado', 'init', 'tests', quiet=True)
name_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'name')
salary_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'salary')
boss_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'boss')
concept = DataConcept(name='Title')
concept.save()
DataConceptField(concept=concept, field=name_field, order=1).save()
DataConceptField(concept=concept, field=salary_field, order=2).save()
DataConceptField(concept=concept, field=boss_field, order=3).save()
self.concept = concept
self.values = ['CEO', 100000, True]
def test_unique_keys(self):
title_name = DataField.objects.get_by_natural_key(
'tests', 'title', 'name')
project_name = DataField.objects.get_by_natural_key(
'tests', 'project', 'name')
concept = DataConcept()
concept.save()
DataConceptField(concept=concept, field=title_name, order=1).save()
DataConceptField(concept=concept, field=project_name, order=2).save()
f = Formatter(concept)
self.assertEqual(
OrderedDict([
('title__name', 'one'),
('project__name', 'two'),
]), f(['one', 'two']))
def test_view(self):
salary_field = DataField.objects.get_by_natural_key('exporting', 'title', 'salary')
salary_concept = DataConcept()
salary_concept.save()
DataConceptField(concept=salary_concept, field=salary_field, order=1).save()
view = DataView(json={'ordering': [[salary_concept.pk, 'desc']]})
query = view.apply(tree=models.Employee).raw()
# Ick..
exporter = export.CSVExporter(view)
exporter.params.insert(0, (RawFormatter(keys=['pk']), 1))
exporter.row_length += 1
buff = exporter.write(query)
buff.seek(0)
lines = buff.read().splitlines()
# Skip the header
self.assertEqual([int(x) for x in lines[1:]], [2, 4, 6, 1, 3, 5])
def test_get_all_unrelated(self):
# Publish unrelated field and concept
f = DataField.objects.get(model_name='unrelated')
f.published = True
f.save()
c = DataConcept(name='Unrelated', published=True)
c.save()
DataConceptField(concept=c, field=f, order=1).save()
# Concept from two unrelated fields.. this will never show.
# TODO don't allow this
c = DataConcept(name='Unrelated', published=True)
c.save()
DataConceptField(concept=c, field=f, order=1).save()
DataConceptField(concept=c, field=self.boss_field, order=2).save()
# Still 2 concepts visible by default tree
response = self.client.get('/api/concepts/',
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, codes.ok)
self.assertEqual(len(json.loads(response.content)), 2)
# 1 concept visible for unrelated tree
response = self.client.get('/api/concepts/?tree=unrelated',
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, codes.ok)
self.assertEqual(len(json.loads(response.content)), 1)
def test_published(self):
concept = DataConcept(published=True)
concept.save()
DataConceptField(concept=concept, field=self.is_manager).save()
DataConceptField(concept=concept, field=self.salary).save()
self.assertEqual([x.pk for x in DataConcept.objects.published()], [])
self.is_manager.published = True
self.is_manager.save()
self.salary.published = True
self.salary.save()
# Now published, it will appear
self.assertEqual([x.pk for x in DataConcept.objects.published()],
[concept.pk])
# Set the category to be an unpublished category and it should no
# longer appear.
concept.category = self.category
concept.save()
self.assertEqual([x.pk for x in DataConcept.objects.published()], [])
# Publish the category and the concept should appear again
self.category.published = True
self.category.save()
self.assertEqual([x.pk for x in DataConcept.objects.published()],
[concept.pk])
user1 = User.objects.create_user('user1', 'user1')
# Nothing since user1 cannot view either datafield
self.assertEqual([x.pk for x in DataConcept.objects.published(user1)],
[])
assign('avocado.view_datafield', user1, self.is_manager)
# Still nothing since user1 has no permission for salary
self.assertEqual([x.pk for x in DataConcept.objects.published(user1)],
[])
assign('avocado.view_datafield', user1, self.salary)
# Now user1 can see the concept
self.assertEqual([x.pk for x in DataConcept.objects.published(user1)],
[concept.pk])
user2 = User.objects.create_user('user2', 'user2')
# `user2` is not assigned
self.assertEqual([x.pk for x in DataConcept.objects.published(user2)],
[])
# Remove the fields from the concept and it should no longer appear
# as published.
DataConceptField.objects.filter(concept=concept).delete()
self.assertEqual([x.pk for x in DataConcept.objects.published()], [])
class BaseTestCase(TestCase):
fixtures = ['tests/fixtures/test_data.json']
def setUp(self):
management.call_command('avocado', 'init', 'tests', quiet=True)
f1 = DataField.objects.get_by_natural_key('tests',
'employee',
'first_name')
f2 = DataField.objects.get_by_natural_key('tests',
'employee',
'last_name')
self.c = DataConcept()
self.c.save()
DataConceptField(concept=self.c, field=f1).save()
DataConceptField(concept=self.c, field=f2).save()
self.request = HttpRequest()
self.request.session = SessionStore()
self.request.session.save()
def test_published(self):
concept = DataConcept(published=True)
concept.save()
DataConceptField(concept=concept, field=self.is_manager).save()
DataConceptField(concept=concept, field=self.salary).save()
self.assertEqual([x.pk for x in DataConcept.objects.published()], [])
self.is_manager.published = True
self.is_manager.save()
self.salary.published = True
self.salary.save()
# Now published, it will appear
self.assertEqual([x.pk for x in DataConcept.objects.published()],
[concept.pk])
# Set the category to be an unpublished category and it should no
# longer appear.
concept.category = self.category
concept.save()
self.assertEqual([x.pk for x in DataConcept.objects.published()], [])
# Publish the category and the concept should appear again
self.category.published = True
self.category.save()
self.assertEqual([x.pk for x in DataConcept.objects.published()],
[concept.pk])
user1 = User.objects.create_user('user1', 'user1')
# Nothing since user1 cannot view either datafield
self.assertEqual(
[x.pk for x in DataConcept.objects.published(user1)], [])
assign('avocado.view_datafield', user1, self.is_manager)
# Still nothing since user1 has no permission for salary
self.assertEqual(
[x.pk for x in DataConcept.objects.published(user1)], [])
assign('avocado.view_datafield', user1, self.salary)
# Now user1 can see the concept
self.assertEqual([x.pk for x in DataConcept.objects.published(user1)],
[concept.pk])
user2 = User.objects.create_user('user2', 'user2')
# `user2` is not assigned
self.assertEqual(
[x.pk for x in DataConcept.objects.published(user2)], [])
# Remove the fields from the concept and it should no longer appear
# as published.
DataConceptField.objects.filter(concept=concept).delete()
self.assertEqual([x.pk for x in DataConcept.objects.published()], [])
def setUp(self):
management.call_command('avocado', 'sync', 'openmrs')
birthdate_field = DataField.objects.get_by_natural_key(
'openmrs', 'patient', 'birthdate')
estimate_field = DataField.objects.get_by_natural_key(
'openmrs', 'patient', 'birthdate_estimated')
self.concept = concept = DataConcept(name="Birthdate")
concept.save()
DataConceptField(concept=concept, field=birthdate_field,
order=1).save()
DataConceptField(concept=concept, field=estimate_field, order=2).save()
formatter = registry.get('Age')
self.f = formatter(concept)
def setUp(self):
management.call_command('avocado', 'init', 'tests', quiet=True)
first_name_field = DataField.objects.get_by_natural_key(
'tests', 'employee', 'first_name')
first_name_field.description = 'First Name'
first_name_field.save()
last_name_field = DataField.objects.get_by_natural_key(
'tests', 'employee', 'last_name')
last_name_field.description = 'Last Name'
last_name_field.save()
title_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'name')
title_field.description = 'Employee Title'
title_field.save()
salary_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'salary')
salary_field.description = 'Salary'
salary_field.save()
is_manager_field = DataField.objects.get_by_natural_key(
'tests', 'employee', 'is_manager')
is_manager_field.description = 'Is a Manager?'
is_manager_field.save()
employee_concept = DataConcept()
employee_concept.name = 'Employee'
employee_concept.description = 'A Single Employee'
employee_concept.save()
DataConceptField(concept=employee_concept,
field=first_name_field,
order=1).save()
DataConceptField(concept=employee_concept,
field=last_name_field,
order=2).save()
DataConceptField(concept=employee_concept,
field=is_manager_field,
order=3).save()
DataConceptField(concept=employee_concept, field=title_field,
order=4).save()
DataConceptField(concept=employee_concept, field=salary_field,
order=5).save()
self.concepts = [employee_concept]
self.query = models.Employee.objects.values_list(
'first_name', 'last_name', 'is_manager', 'title__name',
'title__salary')
def test_format(self):
name_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'name')
salary_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'salary')
boss_field = DataField.objects.get_by_natural_key(
'tests', 'title', 'boss')
concept = DataConcept(name='Title')
concept.save()
DataConceptField(concept=concept, field=name_field, order=1).save()
DataConceptField(concept=concept, field=salary_field, order=2).save()
DataConceptField(concept=concept, field=boss_field, order=3).save()
values = ['CEO', 100000, True]
self.assertEqual(
concept.format(values),
OrderedDict([(u'name', u'CEO'), (u'salary', 100000),
(u'boss', True)]))
self.assertEqual(concept._formatter_cache[0], None)
from avocado.formatters import Formatter, registry as formatters
class HtmlFormatter(Formatter):
def to_html(self, values, **context):
fvalues = self(values, preferred_formats=['string'])
return OrderedDict({
'profile':
'<span>' + '</span><span>'.join(fvalues.values()) +
'</span>'
})
to_html.process_multiple = True
formatters.register(HtmlFormatter, 'HTML')
concept.formatter_name = 'HTML'
self.assertEqual(
concept.format(values, preferred_formats=['html']),
OrderedDict([
('profile',
u'<span>CEO</span><span>100000</span><span>True</span>')
]))
def test_format(self):
name_field = DataField.objects.get_by_natural_key('models', 'title', 'name')
salary_field = DataField.objects.get_by_natural_key('models', 'title', 'salary')
boss_field = DataField.objects.get_by_natural_key('models', 'title', 'boss')
concept = DataConcept(name='Title')
concept.save()
DataConceptField(concept=concept, field=name_field, order=1).save()
DataConceptField(concept=concept, field=salary_field, order=2).save()
DataConceptField(concept=concept, field=boss_field, order=3).save()
values = ['CEO', 100000, True]
self.assertEqual(concept.format(values),
OrderedDict([
(u'name', u'CEO'),
(u'salary', 100000),
(u'boss', True)
]))
self.assertEqual(concept._formatter_cache[0], None)
from avocado.formatters import Formatter, registry as formatters
class HtmlFormatter(Formatter):
def to_html(self, values, **context):
fvalues = self(values, preferred_formats=['string'])
return OrderedDict({
'profile': '<span>' + '</span><span>'.join(fvalues.values()) + '</span>'
})
to_html.process_multiple = True
formatters.register(HtmlFormatter, 'HTML')
concept.formatter_name = 'HTML'
self.assertEqual(concept.format(values, preferred_formats=['html']),
OrderedDict([
('profile', u'<span>CEO</span><span>100000</span><span>True</span>')
]))
def test_field_for_concept(self):
f = DataField.objects.get(id=4)
c1 = DataConcept()
c2 = DataConcept()
c1.save()
c2.save()
cf = DataConceptField(concept=c1, field=f)
cf.save()
self.assertEqual(parsers.datacontext.validate({
'concept': c1.pk,
'field': 4,
'operator': 'exact',
'value': 'CEO'
}, tree=Employee), None)
# Invalid concept
self.assertRaises(ValidationError, parsers.datacontext.validate, {
'concept': c2.pk,
'field': 4,
'operator': 'exact',
'value': 'CEO'
}, tree=Employee)
def setUp(self):
super(FieldValidatorTestCase, self).setUp()
self.field = DataField.objects.get_by_natural_key('tests.title.salary')
self.concept = DataConcept(name='Salary', ident='salary')
self.concept.save()
DataConceptField(concept=self.concept, field=self.field).save()
class FieldValidatorTestCase(ValidatorTestCase):
def setUp(self):
super(FieldValidatorTestCase, self).setUp()
self.field = DataField.objects.get_by_natural_key('tests.title.salary')
self.concept = DataConcept(name='Salary', ident='salary')
self.concept.save()
DataConceptField(concept=self.concept, field=self.field).save()
def test_valid_field_nk(self):
"Field with natural key"
data = {
'field': 'tests.title.salary',
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
def test_valid_field_nk_list(self):
"Field with list-based natural key"
data = {
'field': ['tests', 'title', 'salary'],
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
def test_valid_field_pk(self):
"Field with primary key"
data = {
'field': self.field.pk,
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
def test_valid_concept(self):
"Field with concept primary key"
data = {
'field': self.field.pk,
'concept': self.concept.pk,
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
self.assertEqual(v.cleaned_data['concept'], self.concept)
def test_valid_concept_ident(self):
"Field with concept ident"
data = {
'field': self.field.pk,
'concept': 'salary',
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
self.assertEqual(v.cleaned_data['concept'], self.concept)
def test_valid_concept_short_field(self):
"Field short name with concept"
data = {
'field': 'salary',
'concept': self.concept.pk,
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
self.assertEqual(v.cleaned_data['concept'], self.concept)
def test_invalid_field(self):
"Field does not exist"
v = FieldValidator({
'field': 'invalid.lookup',
})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'field_does_not_exist')
self.assertFalse(v.data['enabled'])
def test_invalid_concept(self):
"Concept does not exist"
v = FieldValidator({
'field': self.field.pk,
'concept': 'does_not_exist',
})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'concept_does_not_exist')
self.assertFalse(v.data['enabled'])
def test_invalid_field_for_concept(self):
"Field not contained in concept"
v = FieldValidator({
'field': 'tests.title.name',
'concept': self.concept.pk,
})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'field_does_not_exist_for_concept')
self.assertFalse(v.data['enabled'])
def test_ambiguous_field(self):
"Ambiguous field"
# Could be the name for salary or project..
v = FieldValidator({
'field': 'name'
})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'ambiguous_field')
self.assertFalse(v.data['enabled'])
def test_get_all_category_sort(self):
# Create some temporary concepts and categories
cat1 = DataCategory(name='Category1', order=1.0, published=True)
cat1.save()
c1 = DataConcept(name='B', published=True, category=cat1)
c1.save()
field1 = DataConceptField(concept=c1, field=self.name_field, order=1)
field1.save()
c2 = DataConcept(name='C', published=True, category=cat1)
c2.save()
field2 = DataConceptField(concept=c2, field=self.name_field, order=1)
field2.save()
c3 = DataConcept(name='A', published=True, category=cat1)
c3.save()
field3 = DataConceptField(concept=c3, field=self.name_field, order=1)
field3.save()
# Check that category ordering is happening by default
response = self.client.get('/api/concepts/',
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['Title', 'Name', 'B', 'C', 'A'])
# Reverse the ordering of the categories
response = self.client.get('/api/concepts/',
{'order': 'desc'},
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['B', 'C', 'A', 'Title', 'Name'])
# Order by concept name in addition to category
response = self.client.get('/api/concepts/',
{'sort': 'name'},
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['Name', 'Title', 'A', 'B', 'C'])
# Reverse the name and category sorting
response = self.client.get('/api/concepts/',
{'sort': 'name', 'order': 'desc'},
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['C', 'B', 'A', 'Title', 'Name'])
c1.delete()
c2.delete()
c3.delete()
field1.delete()
field2.delete()
field3.delete()
cat1.delete()
class FieldValidatorTestCase(ValidatorTestCase):
def setUp(self):
super(FieldValidatorTestCase, self).setUp()
self.field = DataField.objects.get_by_natural_key('tests.title.salary')
self.concept = DataConcept(name='Salary')
self.concept.save()
DataConceptField(concept=self.concept, field=self.field).save()
def test_valid_field_nk(self):
"Field with natural key"
data = {
'field': 'tests.title.salary',
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
def test_valid_field_nk_list(self):
"Field with list-based natural key"
data = {
'field': ['tests', 'title', 'salary'],
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
def test_valid_field_pk(self):
"Field with primary key"
data = {
'field': self.field.pk,
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
def test_valid_concept(self):
"Field with concept primary key"
data = {
'field': self.field.pk,
'concept': self.concept.pk,
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
self.assertEqual(v.cleaned_data['concept'], self.concept)
def test_valid_concept_ident(self):
"Field with concept ident"
data = {
'field': self.field.pk,
'concept': 'salary',
}
v = FieldValidator(data)
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'concept_wrong_format')
self.assertFalse(v.data['enabled'])
def test_valid_concept_short_field(self):
"Field short name with concept"
data = {
'field': 'salary',
'concept': self.concept.pk,
}
v = FieldValidator(data)
self.assertTrue(v.is_valid())
self.assertEqual(v.data, data)
self.assertEqual(v.cleaned_data['field'], self.field)
self.assertEqual(v.cleaned_data['concept'], self.concept)
def test_invalid_field(self):
"Field does not exist"
v = FieldValidator({
'field': 'invalid.lookup',
})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'field_does_not_exist')
self.assertFalse(v.data['enabled'])
def test_invalid_concept(self):
"Concept does not exist"
v = FieldValidator({
'field': self.field.pk,
'concept': -1,
})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'concept_does_not_exist')
self.assertFalse(v.data['enabled'])
def test_invalid_field_for_concept(self):
"Field not contained in concept"
v = FieldValidator({
'field': 'tests.title.name',
'concept': self.concept.pk,
})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'field_does_not_exist_for_concept')
self.assertFalse(v.data['enabled'])
def test_ambiguous_field(self):
"Ambiguous field"
# Could be the name for salary or project..
v = FieldValidator({'field': 'name'})
self.assertFalse(v.is_valid())
self.assertTrue('errors' in v.data)
self.assertEqual(v.errors[0], 'ambiguous_field')
self.assertFalse(v.data['enabled'])
def setUp(self):
super(FieldValidatorTestCase, self).setUp()
self.field = DataField.objects.get_by_natural_key('tests.title.salary')
self.concept = DataConcept(name='Salary')
self.concept.save()
DataConceptField(concept=self.concept, field=self.field).save()
def test_get_all_category_sort(self):
# Create some temporary concepts and categories.
cat1 = DataCategory(name='Category1', order=2.0, published=True)
cat1.save()
c1 = DataConcept(name='B', published=True, category=cat1, order=1)
c1.save()
field1 = DataConceptField(concept=c1, field=self.name_field, order=1)
field1.save()
c2 = DataConcept(name='C', published=True, category=cat1, order=2)
c2.save()
field2 = DataConceptField(concept=c2, field=self.name_field, order=1)
field2.save()
c3 = DataConcept(name='A', published=True, category=cat1, order=3)
c3.save()
field3 = DataConceptField(concept=c3, field=self.name_field, order=1)
field3.save()
# Check that category ordering is happening by default.
response = self.client.get('/api/concepts/',
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, codes.ok)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['Title', 'Name', 'B', 'C', 'A'])
# Reverse the ordering of the categories.
response = self.client.get('/api/concepts/',
{'order': 'desc'},
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, codes.ok)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['B', 'C', 'A', 'Title', 'Name'])
# Order by concept name in addition to category.
response = self.client.get('/api/concepts/',
{'sort': 'name'},
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, codes.ok)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['Name', 'Title', 'A', 'B', 'C'])
# Reverse the name and category sorting.
response = self.client.get('/api/concepts/',
{'sort': 'name', 'order': 'desc'},
HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, codes.ok)
self.assertEqual(len(json.loads(response.content)), 5)
names = [concept.get('name', '') for concept in
json.loads(response.content)]
self.assertEqual(names, ['C', 'B', 'A', 'Title', 'Name'])
c1.delete()
c2.delete()
c3.delete()
field1.delete()
field2.delete()
field3.delete()
cat1.delete()
def _migrate_concept(self, model, migrate_func, **options):
no_input = options.get('no_input')
total_migrated = 0
for lc in model.objects.iterator():
field_nks = list(lc.fields.values('app_name', 'model_name',
'field_name').distinct())
field_cond = Q()
for f in field_nks:
field_cond = field_cond | Q(**f)
fields = DataField.objects.filter(field_cond).distinct()
# Mismatch of fields from new to old
if len(fields) != len(field_nks):
print('One or more fields mismatched for "{0}". '
'Skipping...'.format(lc))
continue
matches = DataConcept.objects.filter(name=lc.name)
# Filter concepts by existence of fields
for f in fields:
matches = matches.filter(fields__app_name=f.app_name,
fields__model_name=f.model_name,
fields__field_name=f.field_name)
num = len(matches)
if num > 1:
print('{0} have the same name and fields. '
'Skipping...'.format(num))
continue
if num == 1:
c = matches[0]
existing = True
if not no_input:
override = True
while True:
response = raw_input(u'Match found for "{0}". '
'Override? [n/Y] '.format(c))
if not response:
break
if response.lower() == 'n':
override = False
break
if not override:
continue
else:
c = DataConcept(queryable=False, viewable=False)
existing = False
c.name = lc.name
c.order = lc.order
c.published = lc.is_public
# This looks odd, but this handles choosing the longer of the two
# descriptions for criterion and column if both exist.
if not c.description or lc.description and \
len(lc.description) > len(c.description):
c.description = lc.description
if lc.category:
try:
kwargs = {
'name__iexact': lc.category.name,
}
# Filter by parent if one exists since categories with the
# same name can exists as sub-categories.
if lc.category.parent_id:
kwargs['parent__name'] = lc.category.parent.name
c.category = DataCategory.objects.get(**kwargs)
except DataCategory.DoesNotExist:
pass
# Apply migration specific function to concept from legacy concept
migrate_func(c, lc)
# Save for foreign key references to concept fields
c.save()
cfs = []
if not existing:
lcfs = list(lc.conceptfields.select_related('field'))
# Dict of legacy concept fields to the new field it
# corresponds to
lcf_map = {}
for lcf in lcfs:
for f in fields:
# Match and break
if lcf.field.natural_key() == f.natural_key():
lcf_map[lcf.pk] = f
break
# Map fields to
# Iterate over all legacy concept fields and create the new
# concept fields
for lcf in lcfs:
f = lcf_map[lcf.pk]
cfs.append(DataConceptField(concept=c, field=f,
name=lcf.name,
order=lcf.order))
# Save concept fields
for cf in cfs:
cf.save()
print 'Migrated "{0}"'.format(c)
total_migrated += 1
print '{0} migrated: {1}'.format(model.__name__, total_migrated)
class DataViewParserTestCase(TestCase):
fixtures = ['tests/fixtures/employee_data.json']
def setUp(self):
management.call_command('avocado', 'init', 'tests', publish=False,
concepts=False, quiet=True)
f1 = DataField.objects.get_by_natural_key('tests',
'employee',
'first_name')
f2 = DataField.objects.get_by_natural_key('tests',
'employee',
'last_name')
self.c = DataConcept()
self.c.save()
DataConceptField(concept=self.c, field=f1).save()
DataConceptField(concept=self.c, field=f2).save()
def test_valid(self):
# Single by id
self.assertEqual(parsers.dataview.validate([{
'concept': self.c.pk
}], tree=Employee), [{
'concept': self.c.pk,
}])
self.assertEqual(parsers.dataview.validate([{
'concept': self.c.pk,
'sort': 'desc',
}], tree=Employee), [{
'concept': self.c.pk,
'sort': 'desc',
}])
def test_valid_legacy(self):
# Single by id
self.assertEqual(parsers.dataview.validate({
'columns': [self.c.pk],
}, tree=Employee), [{
'concept': self.c.pk,
'visible': True,
'sort': None,
'sort_index': None,
}])
self.assertEqual(parsers.dataview.validate({
'ordering': [(self.c.pk, 'desc')],
}, tree=Employee), [{
'concept': self.c.pk,
'visible': False,
'sort': 'desc',
'sort_index': 0,
}])
def test_invalid(self):
# Non-existent data field
facets = parsers.dataview.validate({'columns': [999]})
self.assertFalse(facets[0]['enabled'])
self.assertTrue(facets[0]['errors'])
# Invalid ordering
facets = parsers.dataview.validate([{
'concept': self.c.pk,
'sort': 'foo',
}])
self.assertTrue(facets[0]['warnings'])
def test_apply(self):
node = parsers.dataview.parse([{
'concept': self.c.pk,
}], tree=Employee)
self.assertEqual(
unicode(node.apply().query).replace(' ', ''),
'SELECT "tests_employee"."id", "tests_employee"."first_name", '
'"tests_employee"."last_name" FROM "tests_employee"'
.replace(' ', ''))
node = parsers.dataview.parse([{
'concept': self.c.pk,
'sort': 'desc',
'visible': False,
}], tree=Employee)
self.assertEqual(
unicode(node.apply().query).replace(' ', ''),
'SELECT "tests_employee"."id" FROM "tests_employee" '
'ORDER BY "tests_employee"."first_name" DESC, '
'"tests_employee"."last_name" DESC'
.replace(' ', ''))
def test_apply_distinct(self):
node = parsers.dataview.parse([{
'concept': self.c.pk,
}], tree=Employee)
self.assertEqual(
unicode(node.apply(Employee.objects.distinct()).query)
.replace(' ', ''),
'SELECT DISTINCT "tests_employee"."id", '
'"tests_employee"."first_name", '
'"tests_employee"."last_name" FROM "tests_employee"'
.replace(' ', ''))
def test_implicit_apply_distinct(self):
f1 = DataField.objects.get_by_natural_key('tests',
'office',
'location')
f2 = DataField.objects.get_by_natural_key('tests',
'title',
'name')
c = DataConcept()
c.save()
DataConceptField(concept=c, field=f1).save()
DataConceptField(concept=c, field=f2).save()
# Due to the use of distinct, the concept fields appear in the SELECT
# statement at this point. This is not a bug, but a requirement of SQL.
# These columns are stripped downstream by the exporter.
node = parsers.dataview.parse([{
'concept': c.pk,
'sort': 'desc',
'visible': False,
}], tree=Employee)
self.assertEqual(
unicode(node.apply(Employee.objects.distinct()).query)
.replace(' ', ''),
'SELECT DISTINCT "tests_employee"."id", '
'"tests_office"."location", "tests_title"."name" FROM '
'"tests_employee" INNER JOIN "tests_office" ON '
'("tests_employee"."office_id" = "tests_office"."id") LEFT OUTER '
'JOIN "tests_title" ON ("tests_employee"."title_id" = '
'"tests_title"."id") ORDER BY "tests_office"."location" DESC, '
'"tests_title"."name" DESC'
.replace(' ', ''))
class DataQueryParserTestCase(TestCase):
fixtures = ['employee_data.json']
def setUp(self):
management.call_command('avocado', 'init', 'tests', publish=False,
concepts=False, quiet=True)
f1 = DataField.objects.get_by_natural_key('tests',
'employee',
'first_name')
f2 = DataField.objects.get_by_natural_key('tests',
'employee',
'last_name')
self.c = DataConcept()
self.c.save()
DataConceptField(concept=self.c, field=f1).save()
DataConceptField(concept=self.c, field=f2).save()
def test_valid(self):
self.assertEqual(parsers.dataquery.validate({}, tree=Employee), None)
attrs = {
'context': {
'field': 'tests.title.name',
'operator': 'exact',
'value': 'CEO',
'cleaned_value': {'value': 'CEO', 'label': 'CEO'},
'language': 'Name is CEO'
},
'view': [{
'concept': self.c.pk,
}],
}
exp_attrs = deepcopy(attrs)
self.assertEqual(parsers.dataquery.validate(attrs, tree=Employee),
exp_attrs)
# Only the context
attrs = {
'context': {
'field': 'tests.title.name',
'operator': 'exact',
'value': 'CEO',
'cleaned_value': {
'value': 'CEO',
'label': 'CEO',
},
'language': 'Name is CEO'
}
}
exp_attrs = deepcopy(attrs)
exp_attrs['view'] = None
self.assertEqual(parsers.dataquery.validate(attrs, tree=Employee),
exp_attrs)
# Only the view
attrs = {
'view': [{
'concept': self.c.pk,
'visible': False,
'sort': 'desc',
}]
}
exp_attrs = {
'context': None,
'view': [{
'visible': False,
'concept': self.c.pk,
'sort': 'desc',
}],
}
self.assertEqual(parsers.dataquery.validate(attrs, tree=Employee),
exp_attrs)
def test_parsed_node(self):
# Make sure no context or view subnodes are created
node = parsers.dataquery.parse({}, tree=Employee)
self.assertEqual(node.datacontext_node, None)
self.assertEqual(node.dataview_node, None)
node = parsers.dataquery.parse({
'context': {
'type': 'and',
'children': [],
}
}, tree=Employee)
# No condition has been defined..
self.assertEqual(node.datacontext_node.condition, None)
node = parsers.dataquery.parse({
'context': {
'type': 'and',
'children': [{
'field': 'tests.title.name',
'operator': 'exact',
'value': 'CEO',
}]
}
}, tree=Employee)
# Only the one condition is represented
self.assertEqual(str(node.datacontext_node.condition),
"(AND: ('title__name__exact', u'CEO'))")
def test_apply(self):
node = parsers.dataquery.parse({
'context': {
'field': 'tests.title.boss',
'operator': 'exact',
'value': True
},
'view': [{
'concept': self.c.pk,
}],
}, tree=Employee)
self.assertEqual(
unicode(node.apply().query).replace(' ', ''),
'SELECT DISTINCT "tests_employee"."id", '
'"tests_employee"."first_name", "tests_employee"."last_name" FROM '
'"tests_employee" INNER JOIN "tests_title" ON '
'("tests_employee"."title_id" = "tests_title"."id") '
'WHERE "tests_title"."boss" = True '
.replace(' ', ''))
# Just the view
node = parsers.dataquery.parse({
'view': [{
'concept': self.c.pk,
'sort': 'desc',
}]
}, tree=Employee)
self.assertEqual(
unicode(node.apply().query).replace(' ', ''),
'SELECT DISTINCT "tests_employee"."id", '
'"tests_employee"."first_name", '
'"tests_employee"."last_name" FROM "tests_employee" '
'ORDER BY "tests_employee"."first_name" DESC, '
'"tests_employee"."last_name" DESC'.replace(' ', ''))
# Just the context
node = parsers.dataquery.parse({
'context': {
'field': 'tests.title.boss',
'operator': 'exact',
'value': True
}
}, tree=Employee)
self.assertEqual(
unicode(node.apply().values('id').query).replace(' ', ''),
'SELECT DISTINCT "tests_employee"."id" FROM "tests_employee" '
'INNER JOIN "tests_title" ON ("tests_employee"."title_id" = '
'"tests_title"."id") WHERE "tests_title"."boss" = True '
.replace(' ', ''))
f = DataField.objects.get_by_natural_key('tests', 'title', 'boss')
self.assertEqual(node.datacontext_node.language, {
'operator': 'exact',
'language': u'Boss is True',
'field': f.pk,
'value': True
})
def _migrate_concept(self, model, migrate_func, **options):
no_input = options.get('no_input')
total_migrated = 0
for lc in model.objects.iterator():
field_nks = list(
lc.fields.values('app_name', 'model_name',
'field_name').distinct())
field_cond = Q()
for f in field_nks:
field_cond = field_cond | Q(**f)
fields = DataField.objects.filter(field_cond).distinct()
# Mismatch of fields from new to old
if len(fields) != len(field_nks):
print(
'One or more fields mismatched for "{0}". '
'Skipping...'.format(lc))
continue
matches = DataConcept.objects.filter(name=lc.name)
# Filter concepts by existence of fields
for f in fields:
matches = matches.filter(fields__app_name=f.app_name,
fields__model_name=f.model_name,
fields__field_name=f.field_name)
num = len(matches)
if num > 1:
print('{0} have the same name and fields. '
'Skipping...'.format(num))
continue
if num == 1:
c = matches[0]
existing = True
if not no_input:
override = True
while True:
response = raw_input(u'Match found for "{0}". '
'Override? [n/Y] '.format(c))
if not response:
break
if response.lower() == 'n':
override = False
break
if not override:
continue
else:
c = DataConcept(queryable=False, viewable=False)
existing = False
c.name = lc.name
c.order = lc.order
c.published = lc.is_public
# This looks odd, but this handles choosing the longer of the two
# descriptions for criterion and column if both exist.
if not c.description or lc.description and \
len(lc.description) > len(c.description):
c.description = lc.description
if lc.category:
try:
kwargs = {
'name__iexact': lc.category.name,
}
# Filter by parent if one exists since categories with the
# same name can exists as sub-categories.
if lc.category.parent_id:
kwargs['parent__name'] = lc.category.parent.name
c.category = DataCategory.objects.get(**kwargs)
except DataCategory.DoesNotExist:
pass
# Apply migration specific function to concept from legacy concept
migrate_func(c, lc)
# Save for foreign key references to concept fields
c.save()
cfs = []
if not existing:
lcfs = list(lc.conceptfields.select_related('field'))
# Dict of legacy concept fields to the new field it
# corresponds to
lcf_map = {}
for lcf in lcfs:
for f in fields:
# Match and break
if lcf.field.natural_key() == f.natural_key():
lcf_map[lcf.pk] = f
break
# Map fields to
# Iterate over all legacy concept fields and create the new
# concept fields
for lcf in lcfs:
f = lcf_map[lcf.pk]
cfs.append(
DataConceptField(concept=c,
field=f,
name=lcf.name,
order=lcf.order))
# Save concept fields
for cf in cfs:
cf.save()
print 'Migrated "{0}"'.format(c)
total_migrated += 1
print '{0} migrated: {1}'.format(model.__name__, total_migrated)
