def setupFixtureData(cls):
cls.fixture_data = {
'schedule1': '7',
'schedule2': '14',
'schedule3': '21',
}
cls.data_type = FixtureDataType(
domain=cls.domain,
tag=DAILY_SCHEDULE_FIXTURE_NAME,
name=DAILY_SCHEDULE_FIXTURE_NAME,
fields=[
FixtureTypeField(field_name=SCHEDULE_ID_FIXTURE,
properties=[]),
FixtureTypeField(field_name="doses_per_week", properties=[]),
],
item_attributes=[],
)
cls.data_type.save()
cls.data_items = []
for _id, value in cls.fixture_data.iteritems():
data_item = FixtureDataItem(
domain=cls.domain,
data_type_id=cls.data_type.get_id,
fields={
SCHEDULE_ID_FIXTURE:
FieldList(
field_list=[FixtureItemField(field_value=_id, )]),
"doses_per_week":
FieldList(
field_list=[FixtureItemField(field_value=value, )])
},
item_attributes={},
)
data_item.save()
cls.data_items.append(data_item)
def setUpClass(cls):
super(TestFixtureOrdering, cls).setUpClass()
cls.domain = "TestFixtureOrdering"
cls.user = CommCareUser.create(cls.domain, 'george', '***')
cls.data_type = FixtureDataType(
domain=cls.domain,
tag="houses-of-westeros",
is_global=True,
name="Great Houses of Westeros",
fields=[
FixtureTypeField(field_name="name", properties=[]),
FixtureTypeField(field_name="seat", properties=[]),
FixtureTypeField(field_name="sigil", properties=[]),
],
item_attributes=[],
)
cls.data_type.save()
cls.data_items = [
cls._make_data_item(4, "Tyrell", "Highgarden", "Rose"),
cls._make_data_item(6, "Martell", "Sunspear", "Sun and Spear"),
cls._make_data_item(3, "Lannister", "Casterly Rock", "Lion"),
cls._make_data_item(1, "Targaryen", "Dragonstone", "Dragon"),
cls._make_data_item(5, "Tully", "Riverrun", "Trout"),
cls._make_data_item(2, "Stark", "Winterfell", "Direwolf"),
cls._make_data_item(7, "Baratheon", "Storm's End", "Stag"),
]
def setupFixtureData(cls):
cls.fixture_data = [
{
SCHEDULE_ID_FIXTURE: 'schedule1',
'doses_per_week': '7',
'dose_count_ip_new_patient': '56',
'dose_count_ip_recurring_patient': '84',
'dose_count_cp_new_patient': '112',
'dose_count_cp_recurring_patient': '140',
'dose_count_outcome_due_new_patient': '168',
'dose_count_outcome_due_recurring_patient': '168',
},
{
SCHEDULE_ID_FIXTURE: 'schedule2',
'doses_per_week': '14',
'dose_count_ip_new_patient': '24',
'dose_count_ip_recurring_patient': '36',
'dose_count_cp_new_patient': '54',
'dose_count_cp_recurring_patient': '66',
'dose_count_outcome_due_new_patient': '78',
'dose_count_outcome_due_recurring_patient': '78',
},
{
SCHEDULE_ID_FIXTURE: 'schedule3',
'doses_per_week': '21',
'dose_count_ip_new_patient': '24',
'dose_count_ip_recurring_patient': '36',
'dose_count_cp_new_patient': '54',
'dose_count_cp_recurring_patient': '66',
'dose_count_outcome_due_new_patient': '78',
'dose_count_outcome_due_recurring_patient': '78',
},
]
cls.data_type = FixtureDataType(
domain=cls.domain,
tag=DAILY_SCHEDULE_FIXTURE_NAME,
name=DAILY_SCHEDULE_FIXTURE_NAME,
fields=[
FixtureTypeField(field_name=SCHEDULE_ID_FIXTURE,
properties=[]),
FixtureTypeField(field_name="doses_per_week", properties=[]),
],
item_attributes=[],
)
cls.data_type.save()
cls.data_items = []
for row in cls.fixture_data:
data_item = FixtureDataItem(
domain=cls.domain,
data_type_id=cls.data_type.get_id,
fields={
column_name: FieldList(
field_list=[FixtureItemField(field_value=value, )])
for column_name, value in six.iteritems(row)
},
item_attributes={},
)
data_item.save()
cls.data_items.append(data_item)
def setUpClass(cls):
super(TestLocationOwnership, cls).setUpClass()
cls.tag = "big-mac-index"
data_type = FixtureDataType(
domain=cls.domain,
tag=cls.tag,
name="Big Mac Index",
fields=[
FixtureTypeField(field_name="cost", properties=[]),
FixtureTypeField(field_name="region", properties=[]),
],
item_attributes=[],
)
data_type.save()
def make_data_item(location_name, cost):
"""Make a fixture data item and assign it to location_name"""
data_item = FixtureDataItem(
domain=cls.domain,
data_type_id=data_type.get_id,
fields={
"cost": FieldList(
field_list=[FixtureItemField(
field_value=cost,
properties={},
)]
),
"location_name": FieldList(
field_list=[FixtureItemField(
field_value=location_name,
properties={},
)]
),
},
item_attributes={},
)
data_item.save()
FixtureOwnership(
domain=cls.domain,
owner_id=cls.locations[location_name].location_id,
owner_type='location',
data_item_id=data_item.get_id
).save()
make_data_item('Suffolk', '8')
make_data_item('Boston', '10')
make_data_item('Somerville', '7')
get_fixture_data_types.clear(cls.domain)
cls.no_location_user = CommCareUser.create(cls.domain, 'no_location', '***', None, None)
cls.suffolk_user = CommCareUser.create(cls.domain, 'guy-from-suffolk', '***', None, None)
cls.suffolk_user.set_location(cls.locations['Suffolk'])
cls.boston_user = CommCareUser.create(cls.domain, 'guy-from-boston', '***', None, None)
cls.boston_user.set_location(cls.locations['Boston'])
cls.middlesex_user = CommCareUser.create(cls.domain, 'guy-from-middlesex', '***', None, None)
cls.middlesex_user.set_location(cls.locations['Middlesex'])
def setUp(self):
super(FixtureDataTest, self).setUp()
self.domain = 'qwerty'
self.tag = "district"
delete_all_users()
delete_all_fixture_data_types()
self.data_type = FixtureDataType(
domain=self.domain,
tag=self.tag,
name="Districts",
fields=[
FixtureTypeField(field_name="state_name", properties=[]),
FixtureTypeField(field_name="district_name",
properties=["lang"]),
FixtureTypeField(field_name="district_id", properties=[]),
],
item_attributes=[],
)
self.data_type.save()
self.data_item = FixtureDataItem(
domain=self.domain,
data_type_id=self.data_type.get_id,
fields={
"state_name":
FieldList(field_list=[
FixtureItemField(field_value="Delhi_state", properties={})
]),
"district_name":
FieldList(field_list=[
FixtureItemField(field_value="Delhi_in_HIN",
properties={"lang": "hin"}),
FixtureItemField(field_value="Delhi_in_ENG",
properties={"lang": "eng"})
]),
"district_id":
FieldList(field_list=[
FixtureItemField(field_value="Delhi_id", properties={})
])
},
item_attributes={},
)
self.data_item.save()
self.user = CommCareUser.create(self.domain, 'to_delete', '***', None,
None)
self.fixture_ownership = FixtureOwnership(
domain=self.domain,
owner_id=self.user.get_id,
owner_type='user',
data_item_id=self.data_item.get_id)
self.fixture_ownership.save()
get_fixture_data_types.clear(self.domain)
def setUpClass(cls):
super().setUpClass()
cls.table = FixtureDataType(
domain=cls.domain,
tag='moons',
is_global=True,
fields=[
FixtureTypeField(field_name="name"),
FixtureTypeField(field_name="planet"),
],
)
cls.table.save()
def _setup_fixture_type(self):
self.data_type = FixtureDataType(
domain=self.domain_name,
tag=self.fixture_name,
fields=[
FixtureTypeField(field_name="risk_profile", properties=[]),
FixtureTypeField(field_name="sequence", properties=[]),
FixtureTypeField(field_name="message", properties=[]),
],
item_attributes=[])
self.data_type.save()
self.addCleanup(self.data_type.delete)
self._setup_data_item()
def fixture_type_context():
fixture_type = FixtureDataType(domain=DOMAIN,
tag='dhis2_org_unit',
fields=[
FixtureTypeField(field_name='id',
properties=[]),
FixtureTypeField(field_name='name',
properties=[])
])
fixture_type.save()
try:
yield fixture_type
finally:
fixture_type.delete()
def setUp(self):
self.domain = 'dirty-fields'
self.data_type = FixtureDataType(
domain=self.domain,
tag='dirty_fields',
name="Dirty Fields",
fields=[
FixtureTypeField(field_name="will/crash", properties=[]),
FixtureTypeField(field_name="space cadet", properties=[]),
FixtureTypeField(field_name="yes\\no", properties=[]),
FixtureTypeField(field_name="<with>", properties=[]),
FixtureTypeField(field_name="<crazy / combo><d", properties=[])
],
item_attributes=[],
)
self.data_type.save()
self.data_item = FixtureDataItem(
domain=self.domain,
data_type_id=self.data_type.get_id,
fields={
"will/crash":
FieldList(field_list=[
FixtureItemField(field_value="yep", properties={})
]),
"space cadet":
FieldList(field_list=[
FixtureItemField(field_value="major tom", properties={})
]),
"yes\\no":
FieldList(field_list=[
FixtureItemField(field_value="no, duh", properties={})
]),
"<with>":
FieldList(field_list=[
FixtureItemField(field_value="so fail", properties={})
]),
"<crazy / combo><d":
FieldList(field_list=[
FixtureItemField(field_value="just why", properties={})
]),
"xmlbad":
FieldList(field_list=[
FixtureItemField(field_value="badxml", properties={})
])
},
item_attributes={},
)
self.data_item.save()
def _update_types(patches, domain, data_type_id, data_tag, is_global,
description, transaction):
data_type = FixtureDataType.get(data_type_id)
fields_patches = deepcopy(patches)
old_fields = data_type.fields
new_fixture_fields = []
data_type.tag = data_tag
data_type.is_global = is_global
data_type.description = description
for old_field in old_fields:
patch = fields_patches.pop(old_field.field_name, {})
if not any(patch):
new_fixture_fields.append(old_field)
if "update" in patch:
setattr(old_field, "field_name", patch["update"])
new_fixture_fields.append(old_field)
if "remove" in patch:
continue
new_fields = list(fields_patches.keys())
for new_field_name in new_fields:
patch = fields_patches.pop(new_field_name)
if "is_new" in patch:
new_fixture_fields.append(
FixtureTypeField(field_name=new_field_name, properties=[]))
data_type.fields = new_fixture_fields
transaction.save(data_type)
return data_type
def obj_update(self, bundle, **kwargs):
if 'tag' not in bundle.data:
raise BadRequest("tag must be specified")
try:
bundle.obj = FixtureDataType.get(kwargs['pk'])
except ResourceNotFound:
raise NotFound('Lookup table not found')
if bundle.obj.domain != kwargs['domain']:
raise NotFound('Lookup table not found')
if bundle.obj.tag != bundle.data['tag']:
raise BadRequest("Lookup table tag cannot be changed")
save = False
if 'is_global' in bundle.data:
save = True
bundle.obj.is_global = bundle.data['is_global']
if 'fields' in bundle.data:
save = True
bundle.obj.fields = [
FixtureTypeField.wrap(field)
for field in bundle.data['fields']
]
if 'item_attributes' in bundle.data:
save = True
bundle.obj.item_attributes = bundle.data['item_attributes']
if save:
bundle.obj.save()
return bundle
def obj_update(self, bundle, **kwargs):
if 'tag' not in bundle.data:
raise BadRequest("tag must be specified")
try:
bundle.obj = FixtureDataType.get(kwargs['pk'])
except ResourceNotFound:
raise NotFound('Lookup table not found')
if bundle.obj.domain != kwargs['domain']:
raise NotFound('Lookup table not found')
if bundle.obj.tag != bundle.data['tag']:
raise BadRequest("Lookup table tag cannot be changed")
save = False
if 'is_global' in bundle.data:
save = True
bundle.obj.is_global = bundle.data['is_global']
if 'fields' in bundle.data:
save = True
bundle.obj.fields = [
FixtureTypeField.wrap(field) for field in bundle.data['fields']
]
if 'item_attributes' in bundle.data:
save = True
bundle.obj.item_attributes = bundle.data['item_attributes']
if save:
bundle.obj.save()
return bundle
def update_types(patches, domain, data_type_id, data_tag, is_global,
transaction):
data_type = FixtureDataType.get(data_type_id)
fields_patches = deepcopy(patches)
assert (data_type.doc_type == FixtureDataType._doc_type)
assert (data_type.domain == domain)
old_fields = data_type.fields
new_fixture_fields = []
setattr(data_type, "tag", data_tag)
setattr(data_type, "is_global", is_global)
for old_field in old_fields:
patch = fields_patches.pop(old_field.field_name, {})
if not any(patch):
new_fixture_fields.append(old_field)
if "update" in patch:
setattr(old_field, "field_name", patch["update"])
new_fixture_fields.append(old_field)
if "remove" in patch:
continue
new_fields = fields_patches.keys()
for new_field_name in new_fields:
patch = fields_patches.pop(new_field_name)
if "is_new" in patch:
new_fixture_fields.append(
FixtureTypeField(field_name=new_field_name, properties=[]))
setattr(data_type, "fields", new_fixture_fields)
transaction.save(data_type)
return data_type
def setUp(self):
self.domain = 'qwerty'
self.tag = "district"
self.data_type = FixtureDataType(
domain=self.domain,
tag=self.tag,
name="Districts",
fields=[
FixtureTypeField(field_name="state_name", properties=[]),
FixtureTypeField(field_name="district_name",
properties=["lang"]),
FixtureTypeField(field_name="district_id", properties=[])
])
self.data_type.save()
self.data_item = FixtureDataItem(
domain=self.domain,
data_type_id=self.data_type.get_id,
fields={
"state_name":
FieldList(field_list=[
FixtureItemField(field_value="Delhi_state", properties={})
]),
"district_name":
FieldList(field_list=[
FixtureItemField(field_value="Delhi_in_HIN",
properties={"lang": "hin"}),
FixtureItemField(field_value="Delhi_in_ENG",
properties={"lang": "eng"})
]),
"district_id":
FieldList(field_list=[
FixtureItemField(field_value="Delhi_id", properties={})
])
})
self.data_item.save()
self.user = CommCareUser.create(self.domain, 'to_delete', '***')
self.fixture_ownership = FixtureOwnership(
domain=self.domain,
owner_id=self.user.get_id,
owner_type='user',
data_item_id=self.data_item.get_id)
self.fixture_ownership.save()
def test_update_global_only(self):
other_table = FixtureDataType(
domain=self.domain,
tag='jellyfish',
is_global=False,
fields=[
FixtureTypeField(field_name="genus"),
FixtureTypeField(field_name="species"),
],
)
other_table.save()
clear_fixture_quickcache(self.domain,
get_fixture_data_types(self.domain))
clear_fixture_cache(self.domain)
with self.assertRaises(UnsupportedActionError):
update_fixture(self.domain_link, 'jellyfish')
def create_types(fields_patches, domain, data_tag, is_global, transaction):
data_type = FixtureDataType(
domain=domain,
tag=data_tag,
is_global=is_global,
fields=[FixtureTypeField(field_name=field, properties=[]) for field in fields_patches],
item_attributes=[],
)
transaction.save(data_type)
return data_type
def setUpClass(cls):
super(TestLookupTableItemResource, cls).setUpClass()
cls.data_type = FixtureDataType(domain=cls.domain.name,
tag="lookup_table",
fields=[
FixtureTypeField(
field_name="fixture_property",
properties=["lang", "name"])
],
item_attributes=[])
cls.data_type.save()
def setUp(self):
super(TestLookupTableResource, self).setUp()
self.data_type = FixtureDataType(domain=self.domain.name,
tag="lookup_table",
fields=[
FixtureTypeField(
field_name="fixture_property",
properties=["lang", "name"])
],
item_attributes=[])
self.data_type.save()
def make_data_type(self, name, is_global):
data_type = FixtureDataType(
domain=self.domain,
tag="{}-index".format(name),
is_global=is_global,
name=name.title(),
fields=[
FixtureTypeField(field_name="cost", properties=[]),
],
item_attributes=[],
)
data_type.save()
self.addCleanup(data_type.delete)
return data_type
def _create_fixture(domain, tag="table", should_save=True):
data_type = FixtureDataType(domain=domain,
tag=tag,
fields=[
FixtureTypeField(
field_name="fixture_property",
properties=["test"])
],
item_attributes=[],
is_global=True)
if should_save:
data_type.save()
return data_type
def from_row(cls, row_dict):
tag = row_dict.get('table_id') or row_dict.get('tag')
if tag is None:
raise ExcelMalformatException(
_(FAILURE_MESSAGES['has_no_column']).format(
column_name='table_id'))
field_names = row_dict.get('field')
item_attributes = row_dict.get('property')
if field_names is None and item_attributes is None:
raise ExcelMalformatException(
_(FAILURE_MESSAGES['neither_fields_nor_attributes']).format(
tag=tag))
field_names = [] if field_names is None else field_names
item_attributes = [] if item_attributes is None else item_attributes
def _get_field_properties(prop_key):
if row_dict.has_key(prop_key):
try:
properties = row_dict[prop_key]["property"]
assert isinstance(properties, list)
except (KeyError, AssertionError):
error_message = _(
FAILURE_MESSAGES["wrong_property_syntax"]).format(
prop_key=prop_key, )
raise ExcelMalformatException(error_message)
else:
return properties
else:
return []
fields = [
FixtureTypeField(field_name=field,
properties=_get_field_properties(
'field {count}'.format(count=i + 1)))
for i, field in enumerate(field_names)
]
return FixtureTableDefinition(
table_id=tag,
fields=fields,
item_attributes=item_attributes,
is_global=row_dict.get('is_global', False),
uid=row_dict.get('UID'),
delete=(row_dict.get(DELETE_HEADER) or '').lower() == 'y',
)
def test_delete(self):
data_type = FixtureDataType(domain=self.domain.name,
tag="lookup_table2",
fields=[
FixtureTypeField(
field_name="fixture_property",
properties=["lang", "name"])
],
item_attributes=[])
data_type.save()
self.addCleanup(data_type.delete)
self.assertEqual(2, len(FixtureDataType.by_domain(self.domain.name)))
response = self._assert_auth_post_resource(self.single_endpoint(
data_type._id),
'',
method='DELETE')
self.assertEqual(response.status_code, 204, response.content)
self.assertEqual(1, len(FixtureDataType.by_domain(self.domain.name)))
def _import_fixtures(domain):
for fixture_name, filename in [
('recipes', 'recipes.csv'),
('conv_factors', 'conv_factors.csv'),
('food_list', 'food_list.csv'),
('food_composition_table', 'food_composition_table.csv'),
]:
fields, rows = _read_csv(filename)
data_type = FixtureDataType(
domain=domain,
tag=fixture_name,
fields=[FixtureTypeField(field_name=field) for field in fields],
)
data_type.save()
with IterDB(FixtureDataItem.get_db(), chunksize=1000) as iter_db:
for vals in rows:
fixture_data_item = _mk_fixture_data_item(
domain, data_type._id, fields, vals)
iter_db.save(fixture_data_item)
def test_empty_data_types(self):
empty_data_type = FixtureDataType(
domain=self.domain,
tag='blank',
name="blank",
fields=[
FixtureTypeField(
field_name="name",
properties=[]
),
],
item_attributes=[],
)
empty_data_type.save()
self.addCleanup(empty_data_type.delete)
get_fixture_data_types.clear(self.domain)
fixtures = call_fixture_generator(self.user.to_ota_restore_user())
self.assertEqual(2, len(fixtures))
check_xml_line_by_line(
self,
"""
<f>
<fixture id="item-list:blank" user_id="{0}">
<blank_list/>
</fixture>
<fixture id="item-list:district" user_id="{0}">
<district_list>
<district>
<state_name>Delhi_state</state_name>
<district_name lang="hin">Delhi_in_HIN</district_name>
<district_name lang="eng">Delhi_in_ENG</district_name>
<district_id>Delhi_id</district_id>
</district>
</district_list>
</fixture>
</f>
""".format(self.user.user_id),
'<f>{}\n{}\n</f>'.format(*[ElementTree.tostring(fixture).decode('utf-8') for fixture in fixtures])
)
def from_row(cls, row_dict):
tag = row_dict.get('table_id') or row_dict.get('tag')
if tag is None:
raise ExcelMalformatException(
_(FAILURE_MESSAGES['has_no_column']).format(
column_name='table_id'))
field_names = row_dict.get('field')
if field_names is None:
raise ExcelMalformatException(
_(FAILURE_MESSAGES['has_no_column']).format(
column_name='table_id'))
def _get_field_properties(field, prop_key):
if row_dict.has_key(prop_key):
try:
return row_dict[prop_key]["property"]
except KeyError:
error_message = _(
FAILURE_MESSAGES["wrong_property_syntax"]).format(
prop_key=prop_key, field=field)
raise ExcelMalformatException(error_message)
else:
return []
fields = [
FixtureTypeField(field_name=field,
properties=_get_field_properties(
field, 'field {count}'.format(count=i + 1)))
for i, field in enumerate(field_names)
]
return FixtureTableDefinitition(
table_id=tag,
fields=fields,
is_global=row_dict.get('is_global', False),
uid=row_dict.get('UID'),
delete=(row_dict.get(DELETE_HEADER) or '').lower() == 'y',
)
def make_item_lists(tag, item_name):
data_type = FixtureDataType(
domain=DOMAIN,
tag=tag,
name="Provinces",
fields=[FixtureTypeField(field_name="name", properties=[])],
item_attributes=[],
is_global=True
)
data_type.save()
data_item = FixtureDataItem(
domain=DOMAIN,
data_type_id=data_type.get_id,
fields={
"name": FieldList(
field_list=[FixtureItemField(field_value=item_name, properties={})]
)
},
item_attributes={},
)
data_item.save()
return data_type, data_item
def run_upload(request, domain, workbook, replace=False):
return_val = {
"unknown_groups": [],
"unknown_users": [],
"number_of_fixtures": 0,
}
failure_messages = {
"has_no_column":
"Workbook 'types' has no column '{column_name}'.",
"has_no_field_column":
"Excel-sheet '{tag}' does not contain the column '{field}' "
"as specified in its 'types' definition",
"has_extra_column":
"Excel-sheet '{tag}' has an extra column" +
"'{field}' that's not defined in its 'types' definition",
"wrong_property_syntax":
"Properties should be specified as 'field 1: property 1'. In 'types' sheet, "
+ "'{prop_key}' for field '{field}' is not correctly formatted",
"sheet_has_no_property":
"Excel-sheet '{tag}' does not contain property " +
"'{property}' of the field '{field}' as specified in its 'types' definition",
"sheet_has_extra_property":
"Excel-sheet '{tag}'' has an extra property " +
"'{property}' for the field '{field}' that's not defined in its 'types' definition. Re-check the formatting",
"invalid_field_with_property":
"Fields with attributes should be numbered as 'field: {field} integer",
"invalid_property":
"Attribute should be written as '{field}: {prop} interger'",
"wrong_field_property_combos":
"Number of values for field '{field}' and attribute '{prop}' should be same",
"replace_with_UID":
"Rows shouldn't contain UIDs while using replace option. Excel sheet '{tag}' contains UID in a row."
}
group_memoizer = GroupMemoizer(domain)
data_types = workbook.get_worksheet(title='types')
def _get_or_raise(container, attr):
try:
return container[attr]
except KeyError:
raise ExcelMalformatException(
_(failure_messages["has_no_column"].format(column_name=attr)))
def diff_lists(list_a, list_b):
set_a = set(list_a)
set_b = set(list_b)
not_in_b = set_a.difference(set_b)
not_in_a = set_a.difference(set_a)
return list(not_in_a), list(not_in_b)
number_of_fixtures = -1
with CouchTransaction() as transaction:
fixtures_tags = []
type_sheets = []
for number_of_fixtures, dt in enumerate(data_types):
try:
tag = _get_or_raise(dt, 'table_id')
except ExcelMalformatException:
tag = _get_or_raise(dt, 'tag')
if tag in fixtures_tags:
error_message = "Upload Failed: Lookup-tables should have unique 'table_id'. There are two rows with table_id '{tag}' in 'types' sheet."
raise DuplicateFixtureTagException(
_(error_message.format(tag=tag)))
fixtures_tags.append(tag)
type_sheets.append(dt)
for number_of_fixtures, dt in enumerate(type_sheets):
try:
tag = _get_or_raise(dt, 'table_id')
except ExcelMalformatException:
messages.info(
request,
_("Excel-header 'tag' is renamed as 'table_id' and 'name' header is no longer needed."
))
tag = _get_or_raise(dt, 'tag')
type_definition_fields = _get_or_raise(dt, 'field')
type_fields_with_properties = []
for count, field in enumerate(type_definition_fields):
prop_key = "field " + str(count + 1)
if dt.has_key(prop_key):
try:
property_list = dt[prop_key]["property"]
except KeyError:
error_message = failure_messages[
"wrong_property_syntax"].format(prop_key=prop_key,
field=field)
raise ExcelMalformatException(_(error_message))
else:
property_list = []
field_with_prop = FixtureTypeField(field_name=field,
properties=property_list)
type_fields_with_properties.append(field_with_prop)
new_data_type = FixtureDataType(
domain=domain,
is_global=dt.get('is_global', False),
tag=tag,
fields=type_fields_with_properties,
)
try:
tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
if tagged_fdt:
data_type = tagged_fdt
# support old usage with 'UID'
elif 'UID' in dt and dt['UID']:
data_type = FixtureDataType.get(dt['UID'])
else:
data_type = new_data_type
pass
if replace:
data_type.recursive_delete(transaction)
data_type = new_data_type
data_type.fields = type_fields_with_properties
data_type.is_global = dt.get('is_global', False)
assert data_type.doc_type == FixtureDataType._doc_type
if data_type.domain != domain:
data_type = new_data_type
messages.error(
request,
_("'%(UID)s' is not a valid UID. But the new type is created."
) % {'UID': dt['UID']})
if dt[DELETE_HEADER] == "Y" or dt[DELETE_HEADER] == "y":
data_type.recursive_delete(transaction)
continue
except (ResourceNotFound, KeyError) as e:
data_type = new_data_type
transaction.save(data_type)
data_items = workbook.get_worksheet(data_type.tag)
for sort_key, di in enumerate(data_items):
# Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
item_fields_list = di['field'].keys()
not_in_sheet, not_in_types = diff_lists(
item_fields_list, data_type.fields_without_attributes)
if len(not_in_sheet) > 0:
error_message = failure_messages[
"has_no_field_column"].format(tag=tag,
field=not_in_sheet[0])
raise ExcelMalformatException(_(error_message))
if len(not_in_types) > 0:
error_message = failure_messages[
"has_extra_column"].format(tag=tag,
field=not_in_types[0])
raise ExcelMalformatException(_(error_message))
# check that properties in 'types' sheet vs item-sheet MATCH
for field in data_type.fields:
if len(field.properties) > 0:
sheet_props = di.get(field.field_name, {})
sheet_props_list = sheet_props.keys()
type_props = field.properties
not_in_sheet, not_in_types = diff_lists(
sheet_props_list, type_props)
if len(not_in_sheet) > 0:
error_message = failure_messages[
"sheet_has_no_property"].format(
tag=tag,
property=not_in_sheet[0],
field=field.field_name)
raise ExcelMalformatException(_(error_message))
if len(not_in_types) > 0:
error_message = failure_messages[
"sheet_has_extra_property"].format(
tag=tag,
property=not_in_types[0],
field=field.field_name)
raise ExcelMalformatException(_(error_message))
# check that fields with properties are numbered
if type(di['field'][field.field_name]) != list:
error_message = failure_messages[
"invalid_field_with_property"].format(
field=field.field_name)
raise ExcelMalformatException(_(error_message))
field_prop_len = len(di['field'][field.field_name])
for prop in sheet_props:
if type(sheet_props[prop]) != list:
error_message = failure_messages[
"invalid_property"].format(
field=field.field_name, prop=prop)
raise ExcelMalformatException(_(error_message))
if len(sheet_props[prop]) != field_prop_len:
error_message = failure_messages[
"wrong_field_property_combos"].format(
field=field.field_name, prop=prop)
raise ExcelMalformatException(_(error_message))
# excel format check should have been covered by this line. Can make assumptions about data now
type_fields = data_type.fields
item_fields = {}
for field in type_fields:
# if field doesn't have properties
if len(field.properties) == 0:
item_fields[field.field_name] = FieldList(field_list=[
FixtureItemField(
# using unicode here, to cast ints, and multi-language strings
field_value=unicode(di['field'][
field.field_name]),
properties={})
])
else:
field_list = []
field_prop_combos = di['field'][field.field_name]
prop_combo_len = len(field_prop_combos)
prop_dict = di[field.field_name]
for x in range(0, prop_combo_len):
fix_item_field = FixtureItemField(
field_value=unicode(field_prop_combos[x]),
properties={
prop: unicode(prop_dict[prop][x])
for prop in prop_dict
})
field_list.append(fix_item_field)
item_fields[field.field_name] = FieldList(
field_list=field_list)
new_data_item = FixtureDataItem(domain=domain,
data_type_id=data_type.get_id,
fields=item_fields,
sort_key=sort_key)
try:
if di['UID'] and not replace:
old_data_item = FixtureDataItem.get(di['UID'])
else:
old_data_item = new_data_item
pass
old_data_item.fields = item_fields
if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id:
old_data_item = new_data_item
messages.error(
request,
_("'%(UID)s' is not a valid UID. But the new item is created."
) % {'UID': di['UID']})
assert old_data_item.doc_type == FixtureDataItem._doc_type
if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
old_data_item.recursive_delete(transaction)
continue
except (ResourceNotFound, KeyError):
old_data_item = new_data_item
transaction.save(old_data_item)
old_groups = old_data_item.groups
for group in old_groups:
old_data_item.remove_group(group)
old_users = old_data_item.users
for user in old_users:
old_data_item.remove_user(user)
for group_name in di.get('group', []):
group = group_memoizer.by_name(group_name)
if group:
old_data_item.add_group(group, transaction=transaction)
else:
messages.error(
request,
_("Unknown group: '%(name)s'. But the row is successfully added"
) % {'name': group_name})
for raw_username in di.get('user', []):
try:
username = normalize_username(str(raw_username),
domain)
except ValidationError:
messages.error(
request,
_("Invalid username: '******'. Row is not added")
% {'name': raw_username})
continue
user = CommCareUser.get_by_username(username)
if user:
old_data_item.add_user(user)
else:
messages.error(
request,
_("Unknown user: '******'. But the row is successfully added"
) % {'name': raw_username})
return_val["number_of_fixtures"] = number_of_fixtures + 1
return return_val
def from_row(cls, row_dict):
tag = row_dict.get('table_id') or row_dict.get('tag')
if tag is None:
raise FixtureUploadError([
_(FAILURE_MESSAGES['has_no_column']).format(
column_name='table_id')
])
if is_identifier_invalid(tag):
raise FixtureUploadError(
[_(FAILURE_MESSAGES['invalid_table_id']).format(tag=tag)])
field_names = row_dict.get('field')
item_attributes = row_dict.get('property')
if field_names is None and item_attributes is None:
raise FixtureUploadError([
_(FAILURE_MESSAGES['neither_fields_nor_attributes']).format(
tag=tag)
])
field_names = [] if field_names is None else field_names
item_attributes = [] if item_attributes is None else item_attributes
def _get_field_properties(prop_key):
properties = []
if prop_key in row_dict and 'property' in row_dict[prop_key]:
properties = row_dict[prop_key]["property"]
if not isinstance(properties, list):
error_message = _(
FAILURE_MESSAGES["wrong_property_syntax"]).format(
prop_key=prop_key, )
raise FixtureUploadError([error_message])
return properties
def _get_field_is_indexed(prop_key):
is_indexed = False
if prop_key in row_dict and 'is_indexed' in row_dict[prop_key]:
is_indexed = row_dict[prop_key]["is_indexed"]
if not isinstance(is_indexed, bool):
error_message = _(
FAILURE_MESSAGES["wrong_index_syntax"]).format(
prop_key=prop_key, )
raise FixtureUploadError([error_message])
return is_indexed
def is_number(text):
text = six.text_type(text)
try:
float(text)
return True
except ValueError:
return False
for i, field_name in enumerate(field_names):
if is_number(field_name):
message = _(
FAILURE_MESSAGES['invalid_field_name_numerical']).format(
i=i + 1,
val=field_name,
)
raise FixtureUploadError([message])
fields = [
FixtureTypeField(
field_name=field,
properties=_get_field_properties(
'field {count}'.format(count=i + 1)),
is_indexed=_get_field_is_indexed(
'field {count}'.format(count=i + 1)),
) for i, field in enumerate(field_names)
]
return cls(
table_id=tag,
fields=fields,
item_attributes=item_attributes,
is_global=row_dict.get('is_global', False),
uid=row_dict.get('UID'),
delete=(row_dict.get(DELETE_HEADER) or '').lower() == 'y',
)