def test_update(self): data_item = self._create_data_item() data_item_update = { "data_type_id": self.data_type._id, "fields": { "state_name": { "field_list": [ {"field_value": "Massachusetts", "properties": {"lang": "en"}}, {"field_value": "马萨诸塞", "properties": {"lang": "zh"}}, ] } }, "item_attributes": { "attribute1": "cool_attr_value", } } response = self._assert_auth_post_resource( self.single_endpoint(data_item._id), json.dumps(data_item_update), method="PUT") data_item = FixtureDataItem.get(data_item._id) self.assertEqual(response.status_code, 204) self.assertEqual(data_item.data_type_id, self.data_type._id) self.assertEqual(len(data_item.fields), 1) self.assertEqual(data_item.fields['state_name'].field_list[0].field_value, 'Massachusetts') self.assertEqual(data_item.fields['state_name'].field_list[0].properties, {"lang": "en"}) self.assertEqual(data_item.item_attributes, {"attribute1": "cool_attr_value"})
def obj_update(self, bundle, **kwargs): if 'data_type_id' not in bundle.data: raise BadRequest("data_type_id must be specified") try: bundle.obj = FixtureDataItem.get(kwargs['pk']) except ResourceNotFound: raise NotFound('Lookup table item not found') if bundle.obj.domain != kwargs['domain']: raise NotFound('Lookup table item not found') save = False if 'fields' in bundle.data: save = True bundle.obj.fields = { field_name: FieldList.wrap(field_list) for field_name, field_list in bundle.data['fields'].items() } if 'item_attributes' in bundle.data: save = True bundle.obj.item_attributes = bundle.data['item_attributes'] if save: bundle.obj.save() return bundle
def _process_data_item(domain, replace, data_type, di, item_fields, item_attributes, sort_key): """Processes a FixtureDataItem from its excel upload. Returns a tuple with - (unsaved) FixtureDataItem instance - boolean flag to indiicate if the data item was deleted - a list of errors """ delete = False errors = [] new_data_item = FixtureDataItem(domain=domain, data_type_id=data_type.get_id, fields=item_fields, item_attributes=item_attributes, sort_key=sort_key) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item old_data_item.fields = item_fields old_data_item.item_attributes = item_attributes if (old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id): old_data_item = new_data_item errors.append( _("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']}) if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": delete = True except (ResourceNotFound, KeyError): old_data_item = new_data_item return old_data_item, delete, errors
def generate_lineage(self, leaf_type, leaf_item_id): leaf_fdi = FixtureDataItem.get(leaf_item_id) index = None for i, h in enumerate(self.hierarchy[::-1]): if h["type"] == leaf_type: index = i if index is None: raise Exception( "Could not generate lineage for AsyncDrillableFilter due to a nonexistent leaf_type (%s)" % leaf_type) lineage = [leaf_fdi] for i, h in enumerate(self.full_hierarchy[::-1]): if i < index or i >= len(self.hierarchy) - 1: continue real_index = len(self.hierarchy) - (i + 1) lineage.insert( 0, FixtureDataItem.by_field_value( self.domain, self.data_types(real_index - 1), h["references"], lineage[0].fields_without_attributes[h["parent_ref"]] ).one()) return lineage
def generate_lineage(self, leaf_type, leaf_item_id): leaf_fdi = FixtureDataItem.get(leaf_item_id) index = None for i, h in enumerate(self.hierarchy[::-1]): if h["type"] == leaf_type: index = i if index is None: raise Exception( "Could not generate lineage for AsyncDrillableFilter due to a nonexistent leaf_type (%s)" % leaf_type) lineage = [leaf_fdi] for i, h in enumerate(self.full_hierarchy[::-1]): if i < index or i >= len(self.hierarchy) - 1: continue real_index = len(self.hierarchy) - (i + 1) lineage.insert( 0, FixtureDataItem.by_field_value( self.domain, self.data_types(real_index - 1), h["references"], lineage[0].fields_without_attributes[ h["parent_ref"]]).one()) return lineage
def data_items(request, domain, data_type_id, data_item_id): def prepare_item(item): ret = strip_json(item, disallow=['data_type_id']) if request.GET.get('groups') == 'true': ret['groups'] = [] for group in item.get_groups(): ret['groups'].append(strip_json(group)) if request.GET.get('users') == 'true': ret['users'] = [] for user in item.get_users(): ret['users'].append(prepare_user(user)) return ret if request.method == 'POST' and data_item_id is None: o = FixtureDataItem(domain=domain, data_type_id=data_type_id, **_to_kwargs(request)) o.save() return json_response(strip_json(o, disallow=['data_type_id'])) elif request.method == 'GET' and data_item_id is None: return json_response([ prepare_item(x) for x in sorted(FixtureDataItem.by_data_type(domain, data_type_id), key=lambda x: x.sort_key) ]) elif request.method == 'GET' and data_item_id: try: o = FixtureDataItem.get(data_item_id) except ResourceNotFound: raise Http404() assert(o.domain == domain and o.data_type.get_id == data_type_id) return json_response(prepare_item(o)) elif request.method == 'PUT' and data_item_id: original = FixtureDataItem.get(data_item_id) new = FixtureDataItem(domain=domain, **_to_kwargs(request)) for attr in 'fields',: setattr(original, attr, getattr(new, attr)) original.save() return json_response(strip_json(original, disallow=['data_type_id'])) elif request.method == 'DELETE' and data_item_id: o = FixtureDataItem.get(data_item_id) assert(o.domain == domain and o.data_type.get_id == data_type_id) with CouchTransaction() as transaction: o.recursive_delete(transaction) return json_response({}) else: return HttpResponseBadRequest()
def obj_delete(self, bundle, **kwargs): try: data_item = FixtureDataItem.get(kwargs['pk']) except ResourceNotFound: raise NotFound('Lookup table item not found') with CouchTransaction() as transaction: data_item.recursive_delete(transaction) return ImmediateHttpResponse(response=HttpAccepted())
def report_subtitles(self): if self.needs_filters: return [] subtitles = ["Date range: %s" % self.daterange_display] if self.selected_fixture(): tag, id = self.selected_fixture() location = FixtureDataItem.get(id).fields_without_attributes['%s_name' % tag] subtitles.append('Location: %s' % location) if self.disease: location = FixtureDataItem.get(self.disease[1]).fields_without_attributes['disease_name'] subtitles.append('Disease: %s' % location) if self.test_version: test_version = FixtureDataItem.get(self.test_version[1]).fields_without_attributes['visible_test_name'] subtitles.append('Test Version: %s' % test_version) return subtitles
def get_grouping_name(self, user): """ Get the name of province/cbo/user (depending on what is selected) """ if not self.selected_province(): return FixtureDataItem.get(user).fields_without_attributes['name'] elif not self.selected_cbo(): return Group.get(user).name else: return CommCareUser.get_by_username(user).name
def report_subtitles(self): if self.needs_filters: return [] subtitles = ["Date range: %s" % self.daterange_display] if self.selected_fixture(): tag, id = self.selected_fixture() location = FixtureDataItem.get(id).fields['%s_name' % tag] subtitles.append('Location: %s' % location) if self.disease: location = FixtureDataItem.get(self.disease[1]).fields['disease_name'] subtitles.append('Disease: %s' % location) if self.test_version: test_version = FixtureDataItem.get(self.test_version[1]).fields['visible_test_name'] subtitles.append('Test Version: %s' % test_version) return subtitles
def data_items(request, domain, data_type_id, data_item_id): def prepare_item(item): ret = strip_json(item, disallow=['data_type_id']) if request.GET.get('groups') == 'true': ret['groups'] = [] for group in item.get_groups(): ret['groups'].append(strip_json(group)) if request.GET.get('users') == 'true': ret['users'] = [] for user in item.get_users(): ret['users'].append(prepare_user(user)) return ret if request.method == 'POST' and data_item_id is None: o = FixtureDataItem(domain=domain, data_type_id=data_type_id, **_to_kwargs(request)) o.save() return json_response(strip_json(o, disallow=['data_type_id'])) elif request.method == 'GET' and data_item_id is None: return json_response([prepare_item(x) for x in FixtureDataItem.by_data_type(domain, data_type_id)]) elif request.method == 'GET' and data_item_id: o = FixtureDataItem.get(data_item_id) assert(o.domain == domain and o.data_type.get_id == data_type_id) return json_response(prepare_item(o)) elif request.method == 'PUT' and data_item_id: original = FixtureDataItem.get(data_item_id) new = FixtureDataItem(domain=domain, **_to_kwargs(request)) for attr in 'fields',: setattr(original, attr, getattr(new, attr)) original.save() return json_response(strip_json(original, disallow=['data_type_id'])) elif request.method == 'DELETE' and data_item_id: o = FixtureDataItem.get(data_item_id) assert(o.domain == domain and o.data_type.get_id == data_type_id) o.delete() return json_response({}) else: return HttpResponseBadRequest()
def get_unique_combinations(domain, place_types=None, place=None): if not place_types: return [] if place: place_type = place[0] place = FixtureDataItem.get(place[1]) place_name = place.fields_without_attributes[place_type + '_id'] place_data_types = {} for pt in place_types: place_data_types[pt] = FixtureDataType.by_domain_tag(domain, pt).one() relevant_types = [t for t in reversed(place_types)] base_type = relevant_types[0] if relevant_types else "" fdis = FixtureDataItem.by_data_type( domain, place_data_types[base_type].get_id) if base_type else [] combos = [] for fdi in fdis: if place: if base_type == place_type: if fdi.fields_without_attributes[base_type + '_id'] != place_name: continue else: rel_type_name = fdi.fields_without_attributes.get( place_type + "_id", "") if not rel_type_name: logging.error( "GSID Reports Error: fixture_id: %s -- place_type: %s" % (fdi.get_id, place_type)) continue if rel_type_name.lower() != place_name: continue comb = {} for pt in place_types: if base_type == pt: comb[pt] = str(fdi.fields_without_attributes[pt + '_id']) comb["gps"] = str(fdi.fields_without_attributes["gps"]) else: p_id = fdi.fields_without_attributes.get(pt + "_id", None) if p_id: if place and pt == place_type and p_id != place_name: continue comb[pt] = str(p_id) else: comb[pt] = None combos.append(comb) return combos
def generate_lineage(self, leaf_type, leaf_item_id): leaf_fdi = FixtureDataItem.get(leaf_item_id) for i, h in enumerate(self.hierarchy[::-1]): if h["type"] == leaf_type: index = i lineage = [leaf_fdi] for i, h in enumerate(self.full_hierarchy[::-1]): if i < index or i >= len(self.hierarchy)-1: continue real_index = len(self.hierarchy) - (i+1) lineage.insert(0, FixtureDataItem.by_field_value(self.domain, self.data_types(real_index - 1), h["references"], lineage[0].fields[h["parent_ref"]]).one()) return lineage
def __init__(self, request, base_context=None, domain=None, **kwargs): super(MCBase, self).__init__(request, base_context, domain, **kwargs) assert self.SECTIONS is not None fixture = self.request_params.get('fixture_id', None) if fixture: type_string, id = fixture.split(":") results = FixtureDataType.by_domain_tag(domain, type_string) fixture_type = results.one() fixture_item = FixtureDataItem.get(id) else: fixture_item = None fixture_type = None sqldata = McSqlData(self.SECTIONS, domain, self.datespan, fixture_type, fixture_item) self.data_provider = MCSectionedDataProvider(sqldata)
def config(self): loc = None type = 'loc' if 'fixture_id' in self.request_params and self.request_params.get('fixture_id'): type, id = self.request_params.get('fixture_id').split(":") loc = FixtureDataItem.get(id).fields['name']['field_list'][0]['field_value'] return { 'domain': self.domain, 'startdate': self.datespan.startdate_param_utc, 'enddate': self.datespan.enddate_param_utc, type: loc, 'one': 1, 'zero': 0, 'not': -1, 'empty': '' }
def __init__(self, request, base_context=None, domain=None, **kwargs): super(MCBase, self).__init__(request, base_context, domain, **kwargs) assert self.SECTIONS is not None assert self.format_class is not None fixture = self.request_params.get('fixture_id', None) if fixture: type_string, id = fixture.split(":") results = FixtureDataType.by_domain_tag(domain, type_string) fixture_type = results.one() fixture_item = FixtureDataItem.get(id) else: fixture_item = None fixture_type = None sqldata = McSqlData(self.SECTIONS, self.format_class, domain, self.datespan, fixture_type, fixture_item) self.data_provider = MCSectionedDataProvider(sqldata)
def get_unique_combinations(domain, place_types=None, place=None): if not place_types: return [] if place: place_type = place[0] place = FixtureDataItem.get(place[1]) place_name = place.fields_without_attributes[place_type + '_id'] place_data_types = {} for pt in place_types: place_data_types[pt] = FixtureDataType.by_domain_tag(domain, pt).one() relevant_types = [t for t in reversed(place_types)] base_type = relevant_types[0] if relevant_types else "" fdis = FixtureDataItem.by_data_type(domain, place_data_types[base_type].get_id) if base_type else [] combos = [] for fdi in fdis: if place: if base_type == place_type: if fdi.fields_without_attributes[base_type + '_id'] != place_name: continue else: rel_type_name = fdi.fields_without_attributes.get(place_type+"_id", "") if not rel_type_name: logging.error("GSID Reports Error: fixture_id: %s -- place_type: %s" % (fdi.get_id, place_type)) continue if rel_type_name.lower() != place_name: continue comb = {} for pt in place_types: if base_type == pt: comb[pt] = str(fdi.fields_without_attributes[pt + '_id']) comb["gps"] = str(fdi.fields_without_attributes["gps"]) else: p_id = fdi.fields_without_attributes.get(pt + "_id", None) if p_id: if place and pt == place_type and p_id != place_name: continue comb[pt] = str(p_id) else: comb[pt] = None combos.append(comb) return combos
def test_update(self): data_item = self._create_data_item() data_item_update = { "data_type_id": self.data_type._id, "fields": { "state_name": { "field_list": [ { "field_value": "Massachusetts", "properties": { "lang": "en" } }, { "field_value": "马萨诸塞", "properties": { "lang": "zh" } }, ] } }, "item_attributes": { "attribute1": "cool_attr_value", } } response = self._assert_auth_post_resource( self.single_endpoint(data_item._id), json.dumps(data_item_update), method="PUT") data_item = FixtureDataItem.get(data_item._id) self.assertEqual(response.status_code, 204) self.assertEqual(data_item.data_type_id, self.data_type._id) self.assertEqual(len(data_item.fields), 1) self.assertEqual( data_item.fields['state_name'].field_list[0].field_value, 'Massachusetts') self.assertEqual( data_item.fields['state_name'].field_list[0].properties, {"lang": "en"}) self.assertEqual(data_item.item_attributes, {"attribute1": "cool_attr_value"})
def obj_get_list(self, bundle, **kwargs): domain = kwargs['domain'] parent_id = bundle.request.GET.get("parent_id", None) parent_ref_name = bundle.request.GET.get("parent_ref_name", None) references = bundle.request.GET.get("references", None) child_type = bundle.request.GET.get("child_type", None) type_id = bundle.request.GET.get("fixture_type_id", None) type_tag = bundle.request.GET.get("fixture_type", None) if parent_id and parent_ref_name and child_type and references: parent_fdi = FixtureDataItem.get(parent_id) fdis = list(FixtureDataItem.by_field_value(domain, child_type, parent_ref_name, parent_fdi.fields_without_attributes[references])) elif type_id or type_tag: type_id = type_id or FixtureDataType.by_domain_tag(domain, type_tag).one() fdis = list(FixtureDataItem.by_data_type(domain, type_id)) else: fdis = list(FixtureDataItem.by_domain(domain)) return [convert_fdt(fdi) for fdi in fdis] or []
def data_item_users(request, domain, data_type_id, data_item_id, user_id): data_type = FixtureDataType.get(data_type_id) data_item = FixtureDataItem.get(data_item_id) user = CommCareUser.get(user_id) assert(data_type.doc_type == FixtureDataType._doc_type) assert(data_type.domain == domain) assert(data_item.doc_type == FixtureDataItem._doc_type) assert(data_item.domain == domain) assert(data_item.data_type_id == data_type_id) assert(user.doc_type == CommCareUser._doc_type) assert(user.domain == domain) if request.method == 'POST': data_item.add_user(user) return json_response({}) elif request.method == 'DELETE': data_item.remove_user(user) return json_response({}) else: return HttpResponseBadRequest()
def data_item_groups(request, domain, data_type_id, data_item_id, group_id): data_type = FixtureDataType.get(data_type_id) data_item = FixtureDataItem.get(data_item_id) group = Group.get(group_id) assert(data_type.doc_type == FixtureDataType._doc_type) assert(data_type.domain == domain) assert(data_item.doc_type == FixtureDataItem._doc_type) assert(data_item.domain == domain) assert(data_item.data_type_id == data_type_id) assert(group.doc_type == Group._doc_type) assert(group.domain == domain) if request.method == 'POST': data_item.add_group(group) return json_response({}) elif request.method == 'DELETE': data_item.remove_group(group) return json_response({}) else: return HttpResponseBadRequest()
def obj_get_list(self, bundle, **kwargs): domain = kwargs['domain'] parent_id = bundle.request.GET.get("parent_id", None) parent_ref_name = bundle.request.GET.get("parent_ref_name", None) references = bundle.request.GET.get("references", None) child_type = bundle.request.GET.get("child_type", None) type_id = bundle.request.GET.get("fixture_type_id", None) type_tag = bundle.request.GET.get("fixture_type", None) if parent_id and parent_ref_name and child_type and references: parent_fdi = FixtureDataItem.get(parent_id) fdis = list( FixtureDataItem.by_field_value( domain, child_type, parent_ref_name, parent_fdi.fields_without_attributes[references])) elif type_id or type_tag: type_id = type_id or FixtureDataType.by_domain_tag( domain, type_tag).one() fdis = list(FixtureDataItem.by_data_type(domain, type_id)) else: fdis = list(FixtureDataItem.by_domain(domain)) return [convert_fdt(fdi) for fdi in fdis] or []
def _process_data_item(domain, replace, data_type, di, item_fields, item_attributes, sort_key): """Processes a FixtureDataItem from its excel upload. Returns a tuple with - (unsaved) FixtureDataItem instance - boolean flag to indiicate if the data item was deleted - a list of errors """ delete = False errors = [] new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=item_fields, item_attributes=item_attributes, sort_key=sort_key ) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item old_data_item.fields = item_fields old_data_item.item_attributes = item_attributes if (old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id): old_data_item = new_data_item errors.append( _("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']} ) if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": delete = True except (ResourceNotFound, KeyError): old_data_item = new_data_item return old_data_item, delete, errors
def run_upload(request, domain, workbook, replace=False): return_val = { "unknown_groups": [], "unknown_users": [], "number_of_fixtures": 0, } failure_messages = { "has_no_column": "Workbook 'types' has no column '{column_name}'.", "has_no_field_column": "Excel-sheet '{tag}' does not contain the column '{field}' " "as specified in its 'types' definition", "has_extra_column": "Excel-sheet '{tag}' has an extra column" + "'{field}' that's not defined in its 'types' definition", "wrong_property_syntax": "Properties should be specified as 'field 1: property 1'. In 'types' sheet, " + "'{prop_key}' for field '{field}' is not correctly formatted", "sheet_has_no_property": "Excel-sheet '{tag}' does not contain property " + "'{property}' of the field '{field}' as specified in its 'types' definition", "sheet_has_extra_property": "Excel-sheet '{tag}'' has an extra property " + "'{property}' for the field '{field}' that's not defined in its 'types' definition. Re-check the formatting", "invalid_field_with_property": "Fields with attributes should be numbered as 'field: {field} integer", "invalid_property": "Attribute should be written as '{field}: {prop} interger'", "wrong_field_property_combos": "Number of values for field '{field}' and attribute '{prop}' should be same", "replace_with_UID": "Rows shouldn't contain UIDs while using replace option. Excel sheet '{tag}' contains UID in a row." } group_memoizer = GroupMemoizer(domain) data_types = workbook.get_worksheet(title='types') def _get_or_raise(container, attr): try: return container[attr] except KeyError: raise ExcelMalformatException( _(failure_messages["has_no_column"].format(column_name=attr))) def diff_lists(list_a, list_b): set_a = set(list_a) set_b = set(list_b) not_in_b = set_a.difference(set_b) not_in_a = set_a.difference(set_a) return list(not_in_a), list(not_in_b) number_of_fixtures = -1 with CouchTransaction() as transaction: fixtures_tags = [] type_sheets = [] for number_of_fixtures, dt in enumerate(data_types): try: tag = _get_or_raise(dt, 'table_id') except ExcelMalformatException: tag = _get_or_raise(dt, 'tag') if tag in fixtures_tags: error_message = "Upload Failed: Lookup-tables should have unique 'table_id'. There are two rows with table_id '{tag}' in 'types' sheet." raise DuplicateFixtureTagException( _(error_message.format(tag=tag))) fixtures_tags.append(tag) type_sheets.append(dt) for number_of_fixtures, dt in enumerate(type_sheets): try: tag = _get_or_raise(dt, 'table_id') except ExcelMalformatException: messages.info( request, _("Excel-header 'tag' is renamed as 'table_id' and 'name' header is no longer needed." )) tag = _get_or_raise(dt, 'tag') type_definition_fields = _get_or_raise(dt, 'field') type_fields_with_properties = [] for count, field in enumerate(type_definition_fields): prop_key = "field " + str(count + 1) if dt.has_key(prop_key): try: property_list = dt[prop_key]["property"] except KeyError: error_message = failure_messages[ "wrong_property_syntax"].format(prop_key=prop_key, field=field) raise ExcelMalformatException(_(error_message)) else: property_list = [] field_with_prop = FixtureTypeField(field_name=field, properties=property_list) type_fields_with_properties.append(field_with_prop) new_data_type = FixtureDataType( domain=domain, is_global=dt.get('is_global', False), tag=tag, fields=type_fields_with_properties, ) try: tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag) if tagged_fdt: data_type = tagged_fdt # support old usage with 'UID' elif 'UID' in dt and dt['UID']: data_type = FixtureDataType.get(dt['UID']) else: data_type = new_data_type pass if replace: data_type.recursive_delete(transaction) data_type = new_data_type data_type.fields = type_fields_with_properties data_type.is_global = dt.get('is_global', False) assert data_type.doc_type == FixtureDataType._doc_type if data_type.domain != domain: data_type = new_data_type messages.error( request, _("'%(UID)s' is not a valid UID. But the new type is created." ) % {'UID': dt['UID']}) if dt[DELETE_HEADER] == "Y" or dt[DELETE_HEADER] == "y": data_type.recursive_delete(transaction) continue except (ResourceNotFound, KeyError) as e: data_type = new_data_type transaction.save(data_type) data_items = workbook.get_worksheet(data_type.tag) for sort_key, di in enumerate(data_items): # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH item_fields_list = di['field'].keys() not_in_sheet, not_in_types = diff_lists( item_fields_list, data_type.fields_without_attributes) if len(not_in_sheet) > 0: error_message = failure_messages[ "has_no_field_column"].format(tag=tag, field=not_in_sheet[0]) raise ExcelMalformatException(_(error_message)) if len(not_in_types) > 0: error_message = failure_messages[ "has_extra_column"].format(tag=tag, field=not_in_types[0]) raise ExcelMalformatException(_(error_message)) # check that properties in 'types' sheet vs item-sheet MATCH for field in data_type.fields: if len(field.properties) > 0: sheet_props = di.get(field.field_name, {}) sheet_props_list = sheet_props.keys() type_props = field.properties not_in_sheet, not_in_types = diff_lists( sheet_props_list, type_props) if len(not_in_sheet) > 0: error_message = failure_messages[ "sheet_has_no_property"].format( tag=tag, property=not_in_sheet[0], field=field.field_name) raise ExcelMalformatException(_(error_message)) if len(not_in_types) > 0: error_message = failure_messages[ "sheet_has_extra_property"].format( tag=tag, property=not_in_types[0], field=field.field_name) raise ExcelMalformatException(_(error_message)) # check that fields with properties are numbered if type(di['field'][field.field_name]) != list: error_message = failure_messages[ "invalid_field_with_property"].format( field=field.field_name) raise ExcelMalformatException(_(error_message)) field_prop_len = len(di['field'][field.field_name]) for prop in sheet_props: if type(sheet_props[prop]) != list: error_message = failure_messages[ "invalid_property"].format( field=field.field_name, prop=prop) raise ExcelMalformatException(_(error_message)) if len(sheet_props[prop]) != field_prop_len: error_message = failure_messages[ "wrong_field_property_combos"].format( field=field.field_name, prop=prop) raise ExcelMalformatException(_(error_message)) # excel format check should have been covered by this line. Can make assumptions about data now type_fields = data_type.fields item_fields = {} for field in type_fields: # if field doesn't have properties if len(field.properties) == 0: item_fields[field.field_name] = FieldList(field_list=[ FixtureItemField( # using unicode here, to cast ints, and multi-language strings field_value=unicode(di['field'][ field.field_name]), properties={}) ]) else: field_list = [] field_prop_combos = di['field'][field.field_name] prop_combo_len = len(field_prop_combos) prop_dict = di[field.field_name] for x in range(0, prop_combo_len): fix_item_field = FixtureItemField( field_value=unicode(field_prop_combos[x]), properties={ prop: unicode(prop_dict[prop][x]) for prop in prop_dict }) field_list.append(fix_item_field) item_fields[field.field_name] = FieldList( field_list=field_list) new_data_item = FixtureDataItem(domain=domain, data_type_id=data_type.get_id, fields=item_fields, sort_key=sort_key) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item pass old_data_item.fields = item_fields if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id: old_data_item = new_data_item messages.error( request, _("'%(UID)s' is not a valid UID. But the new item is created." ) % {'UID': di['UID']}) assert old_data_item.doc_type == FixtureDataItem._doc_type if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": old_data_item.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.groups for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.users for user in old_users: old_data_item.remove_user(user) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: messages.error( request, _("Unknown group: '%(name)s'. But the row is successfully added" ) % {'name': group_name}) for raw_username in di.get('user', []): try: username = normalize_username(str(raw_username), domain) except ValidationError: messages.error( request, _("Invalid username: '******'. Row is not added") % {'name': raw_username}) continue user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: messages.error( request, _("Unknown user: '******'. But the row is successfully added" ) % {'name': raw_username}) return_val["number_of_fixtures"] = number_of_fixtures + 1 return return_val
def columns(self): sum_fn = lambda x, y: int(x or 0) + int(y or 0) total_percent_agg_fn = lambda f_pos, m_pos, f_tot, m_tot: dict(sort_key=sum_fn(f_pos, m_pos), html="%(x)s (%(p)s%%)" % \ { "x": sum_fn(f_pos, m_pos), "p": (100 * sum_fn(f_pos, m_pos) / (sum_fn(m_tot, f_tot) or 1)) }) patient_number_group = DataTablesColumnGroup("Tests") positive_group = DataTablesColumnGroup("Positive Tests") age_range_group = DataTablesColumnGroup("Age Range") male_filter = EQ("gender", "male") female_filter = EQ("gender", "female") columns = self.common_columns + [ DatabaseColumn("Number of Males ", CountColumn('doc_id', alias="male-total", filters=self.filters + [male_filter]), header_group=patient_number_group), DatabaseColumn("Number of Females ", CountColumn('doc_id', alias="female-total", filters=self.filters + [female_filter]), header_group=patient_number_group), AggregateColumn( "Total", sum_fn, [AliasColumn("male-total"), AliasColumn("female-total")], header_group=patient_number_group), AggregateColumn( "Male +ve Percent", self.percent_agg_fn, [ CountColumn( 'doc_id', alias="male-positive", filters=self.filters + [AND([male_filter, EQ("diagnosis", "positive")])]), AliasColumn("male-total") ], header_group=positive_group, sort_type=DTSortType.NUMERIC), AggregateColumn( "Female +ve Percent", self.percent_agg_fn, [ CountColumn( 'doc_id', alias="female-positive", filters=self.filters + [AND([female_filter, EQ("diagnosis", "positive")])]), AliasColumn("female-total") ], header_group=positive_group, sort_type=DTSortType.NUMERIC), AggregateColumn("Total +ve Percent", total_percent_agg_fn, [ AliasColumn("female-positive"), AliasColumn("male-positive"), AliasColumn("female-total"), AliasColumn("male-total") ], header_group=positive_group, sort_type=DTSortType.NUMERIC), AggregateColumn( "Male age range", functools.partial(self.age_fn, 'male'), [ MinColumn("age", alias="male-min", filters=self.filters + [male_filter]), MaxColumn("age", alias="male-max", filters=self.filters + [male_filter]) ], header_group=age_range_group), AggregateColumn( "Female age range", functools.partial(self.age_fn, 'female'), [ MinColumn("age", alias="female-min", filters=self.filters + [female_filter]), MaxColumn("age", alias="female-max", filters=self.filters + [female_filter]) ], header_group=age_range_group), AggregateColumn("All age range", functools.partial(self.age_fn, 'total'), [ MinColumn("age", alias="age-min", filters=self.filters + [OR([female_filter, male_filter])]), MaxColumn("age", alias="age-max", filters=self.filters + [OR([female_filter, male_filter])]) ], header_group=age_range_group), ] if self.is_map: columns.append( DatabaseColumn("gps", MaxColumn(self.gps_key), format_fn=lambda x: x)) disease = FixtureDataItem.get( self.disease[1]).fields_without_attributes[ 'disease_name'] if self.disease else 'All diseases' columns.append( DatabaseColumn('disease', StaticColumn('disease', disease))) return columns
def _run_fixture_upload(domain, workbook, replace=False, task=None): from corehq.apps.users.bulkupload import GroupMemoizer return_val = FixtureUploadResult() group_memoizer = GroupMemoizer(domain) get_location = get_memoized_location_getter(domain) data_types = [] with CouchTransaction() as transaction: type_sheets = workbook.get_all_type_sheets() total_tables = len(type_sheets) return_val.number_of_fixtures = total_tables def _update_progress(table_count, item_count, items_in_table): if task: processed = table_count * 10 + (10 * item_count / items_in_table) DownloadBase.set_progress(task, processed, 10 * total_tables) for table_number, table_def in enumerate(type_sheets): tag = table_def.table_id new_data_type = FixtureDataType( domain=domain, is_global=table_def.is_global, tag=tag, fields=table_def.fields, item_attributes=table_def.item_attributes ) try: tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag) if tagged_fdt: data_type = tagged_fdt # support old usage with 'UID' elif table_def.uid: data_type = FixtureDataType.get(table_def.uid) else: data_type = new_data_type if replace and data_type != new_data_type: data_type.recursive_delete(transaction) data_type = new_data_type data_type.fields = table_def.fields data_type.item_attributes = table_def.item_attributes data_type.is_global = table_def.is_global assert data_type.doc_type == FixtureDataType._doc_type if data_type.domain != domain: data_type = new_data_type return_val.errors.append( _("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': table_def.uid} ) if table_def.delete: data_type.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): data_type = new_data_type transaction.save(data_type) data_types.append(data_type) data_items = list(workbook.get_data_sheet(data_type.tag)) items_in_table = len(data_items) for sort_key, di in enumerate(data_items): _update_progress(table_number, sort_key, items_in_table) type_fields = data_type.fields item_fields = {} for field in type_fields: # if field doesn't have properties if len(field.properties) == 0: item_fields[field.field_name] = FieldList( field_list=[FixtureItemField( # using unicode here, to cast ints, and multi-language strings field_value=six.text_type(di['field'][field.field_name]), properties={} )] ) else: field_list = [] field_prop_combos = di['field'][field.field_name] prop_combo_len = len(field_prop_combos) prop_dict = di[field.field_name] for x in range(0, prop_combo_len): fix_item_field = FixtureItemField( field_value=six.text_type(field_prop_combos[x]), properties={prop: six.text_type(prop_dict[prop][x]) for prop in prop_dict} ) field_list.append(fix_item_field) item_fields[field.field_name] = FieldList( field_list=field_list ) item_attributes = di.get('property', {}) new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=item_fields, item_attributes=item_attributes, sort_key=sort_key ) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item pass old_data_item.fields = item_fields old_data_item.item_attributes = item_attributes if old_data_item.domain != domain \ or not old_data_item.data_type_id == data_type.get_id: old_data_item = new_data_item return_val.errors.append( _("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']} ) assert old_data_item.doc_type == FixtureDataItem._doc_type if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": old_data_item.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.groups for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.users for user in old_users: old_data_item.remove_user(user) old_locations = old_data_item.locations for location in old_locations: old_data_item.remove_location(location) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: return_val.errors.append( _("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name} ) for raw_username in di.get('user', []): try: username = normalize_username(str(raw_username), domain) except ValidationError: return_val.errors.append( _("Invalid username: '******'. Row is not added") % {'name': raw_username} ) continue user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: return_val.errors.append( _("Unknown user: '******'. But the row is successfully added") % {'name': raw_username} ) for name in di.get('location', []): location_cache = get_location(name) if location_cache.is_error: return_val.errors.append(location_cache.message) else: old_data_item.add_location(location_cache.location, transaction=transaction) clear_fixture_quickcache(domain, data_types) clear_fixture_cache(domain) return return_val
def columns(self): sum_fn = lambda x, y: int(x or 0) + int(y or 0) total_percent_agg_fn = lambda f_pos, m_pos, f_tot, m_tot: dict(sort_key=sum_fn(f_pos, m_pos), html="%(x)s (%(p)s%%)" % \ { "x": sum_fn(f_pos, m_pos), "p": (100 * sum_fn(f_pos, m_pos) / (sum_fn(m_tot, f_tot) or 1)) }) patient_number_group = DataTablesColumnGroup("Tests") positive_group = DataTablesColumnGroup("Positive Tests") age_range_group = DataTablesColumnGroup("Age Range") male_filter = EQ("gender", "male") female_filter = EQ("gender", "female") columns = self.common_columns + [ DatabaseColumn( "Number of Males ", SumColumn('cases', alias="male-total", filters=self.filters + [male_filter]), header_group=patient_number_group ), DatabaseColumn( "Number of Females ", SumColumn('cases', alias="female-total", filters=self.filters + [female_filter]), header_group=patient_number_group ), AggregateColumn( "Total", sum_fn, [AliasColumn("male-total"), AliasColumn("female-total")], header_group=patient_number_group ), AggregateColumn( "Male +ve Percent", self.percent_agg_fn, [ SumColumn( 'cases', alias="male-positive", filters=self.filters + [AND([male_filter, EQ("diagnosis", "positive")])] ), AliasColumn("male-total") ], header_group=positive_group, sort_type=DTSortType.NUMERIC ), AggregateColumn( "Female +ve Percent", self.percent_agg_fn, [ SumColumn('cases', alias="female-positive", filters=self.filters + [AND([female_filter, EQ("diagnosis", "positive")])] ), AliasColumn("female-total") ], header_group=positive_group, sort_type=DTSortType.NUMERIC ), AggregateColumn( "Total +ve Percent", total_percent_agg_fn, [ AliasColumn("female-positive"), AliasColumn("male-positive"), AliasColumn("female-total"), AliasColumn("male-total") ], header_group=positive_group, sort_type=DTSortType.NUMERIC ), AggregateColumn( "Male age range", functools.partial(self.age_fn, 'male'), [ MinColumn("age", alias="male-min", filters=self.filters + [male_filter]), MaxColumn("age", alias="male-max", filters=self.filters + [male_filter]) ], header_group=age_range_group ), AggregateColumn( "Female age range", functools.partial(self.age_fn, 'female'), [ MinColumn("age", alias="female-min", filters=self.filters + [female_filter]), MaxColumn("age", alias="female-max", filters=self.filters + [female_filter]) ], header_group=age_range_group ), AggregateColumn( "All age range", functools.partial(self.age_fn, 'total'), [ MinColumn("age", alias="age-min", filters=self.filters + [OR([female_filter, male_filter])]), MaxColumn("age", alias="age-max", filters=self.filters + [OR([female_filter, male_filter])]) ], header_group=age_range_group ), ] if self.is_map: columns.append(DatabaseColumn("gps", MaxColumn(self.gps_key), format_fn=lambda x: x)) disease = FixtureDataItem.get(self.disease[1]).fields['disease_name'] if self.disease else 'All diseases' columns.append(DatabaseColumn('disease', StaticColumn('disease', disease))) return columns
def run_upload(request, domain, workbook, replace=False): return_val = { "unknown_groups": [], "unknown_users": [], "number_of_fixtures": 0, } failure_messages = { "has_no_column": "Workbook 'types' has no column '{column_name}'.", "has_no_field_column": "Excel-sheet '{tag}' does not contain the column '{field}' " "as specified in its 'types' definition", "has_extra_column": "Excel-sheet '{tag}' has an extra column" + "'{field}' that's not defined in its 'types' definition", "wrong_property_syntax": "Properties should be specified as 'field 1: property 1'. In 'types' sheet, " + "'{prop_key}' for field '{field}' is not correctly formatted", "sheet_has_no_property": "Excel-sheet '{tag}' does not contain property " + "'{property}' of the field '{field}' as specified in its 'types' definition", "sheet_has_extra_property": "Excel-sheet '{tag}'' has an extra property " + "'{property}' for the field '{field}' that's not defined in its 'types' definition. Re-check the formatting", "invalid_field_with_property": "Fields with attributes should be numbered as 'field: {field} integer", "invalid_property": "Attribute should be written as '{field}: {prop} interger'", "wrong_field_property_combos": "Number of values for field '{field}' and attribute '{prop}' should be same", "replace_with_UID": "Rows shouldn't contain UIDs while using replace option. Excel sheet '{tag}' contains UID in a row." } group_memoizer = GroupMemoizer(domain) data_types = workbook.get_worksheet(title='types') def _get_or_raise(container, attr): try: return container[attr] except KeyError: raise ExcelMalformatException(_(failure_messages["has_no_column"].format(column_name=attr))) def diff_lists(list_a, list_b): set_a = set(list_a) set_b = set(list_b) not_in_b = set_a.difference(set_b) not_in_a = set_a.difference(set_a) return list(not_in_a), list(not_in_b) number_of_fixtures = -1 with CouchTransaction() as transaction: fixtures_tags = [] type_sheets = [] for number_of_fixtures, dt in enumerate(data_types): try: tag = _get_or_raise(dt, 'table_id') except ExcelMalformatException: tag = _get_or_raise(dt, 'tag') if tag in fixtures_tags: error_message = "Upload Failed: Lookup-tables should have unique 'table_id'. There are two rows with table_id '{tag}' in 'types' sheet." raise DuplicateFixtureTagException(_(error_message.format(tag=tag))) fixtures_tags.append(tag) type_sheets.append(dt) for number_of_fixtures, dt in enumerate(type_sheets): try: tag = _get_or_raise(dt, 'table_id') except ExcelMalformatException: messages.info(request, _("Excel-header 'tag' is renamed as 'table_id' and 'name' header is no longer needed.")) tag = _get_or_raise(dt, 'tag') type_definition_fields = _get_or_raise(dt, 'field') type_fields_with_properties = [] for count, field in enumerate(type_definition_fields): prop_key = "field " + str(count + 1) if dt.has_key(prop_key): try: property_list = dt[prop_key]["property"] except KeyError: error_message = failure_messages["wrong_property_syntax"].format( prop_key=prop_key, field=field ) raise ExcelMalformatException(_(error_message)) else: property_list = [] field_with_prop = FixtureTypeField( field_name=field, properties=property_list ) type_fields_with_properties.append(field_with_prop) new_data_type = FixtureDataType( domain=domain, is_global=dt.get('is_global', False), tag=tag, fields=type_fields_with_properties, ) try: tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag) if tagged_fdt: data_type = tagged_fdt # support old usage with 'UID' elif 'UID' in dt and dt['UID']: data_type = FixtureDataType.get(dt['UID']) else: data_type = new_data_type pass if replace: data_type.recursive_delete(transaction) data_type = new_data_type data_type.fields = type_fields_with_properties data_type.is_global = dt.get('is_global', False) assert data_type.doc_type == FixtureDataType._doc_type if data_type.domain != domain: data_type = new_data_type messages.error(request, _("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': dt['UID']}) if dt[DELETE_HEADER] == "Y" or dt[DELETE_HEADER] == "y": data_type.recursive_delete(transaction) continue except (ResourceNotFound, KeyError) as e: data_type = new_data_type transaction.save(data_type) data_items = workbook.get_worksheet(data_type.tag) for sort_key, di in enumerate(data_items): # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH item_fields_list = di['field'].keys() not_in_sheet, not_in_types = diff_lists(item_fields_list, data_type.fields_without_attributes) if len(not_in_sheet) > 0: error_message = failure_messages["has_no_field_column"].format(tag=tag, field=not_in_sheet[0]) raise ExcelMalformatException(_(error_message)) if len(not_in_types) > 0: error_message = failure_messages["has_extra_column"].format(tag=tag, field=not_in_types[0]) raise ExcelMalformatException(_(error_message)) # check that properties in 'types' sheet vs item-sheet MATCH for field in data_type.fields: if len(field.properties) > 0: sheet_props = di.get(field.field_name, {}) sheet_props_list = sheet_props.keys() type_props = field.properties not_in_sheet, not_in_types = diff_lists(sheet_props_list, type_props) if len(not_in_sheet) > 0: error_message = failure_messages["sheet_has_no_property"].format( tag=tag, property=not_in_sheet[0], field=field.field_name ) raise ExcelMalformatException(_(error_message)) if len(not_in_types) > 0: error_message = failure_messages["sheet_has_extra_property"].format( tag=tag, property=not_in_types[0], field=field.field_name ) raise ExcelMalformatException(_(error_message)) # check that fields with properties are numbered if type(di['field'][field.field_name]) != list: error_message = failure_messages["invalid_field_with_property"].format(field=field.field_name) raise ExcelMalformatException(_(error_message)) field_prop_len = len(di['field'][field.field_name]) for prop in sheet_props: if type(sheet_props[prop]) != list: error_message = failure_messages["invalid_property"].format( field=field.field_name, prop=prop ) raise ExcelMalformatException(_(error_message)) if len(sheet_props[prop]) != field_prop_len: error_message = failure_messages["wrong_field_property_combos"].format( field=field.field_name, prop=prop ) raise ExcelMalformatException(_(error_message)) # excel format check should have been covered by this line. Can make assumptions about data now type_fields = data_type.fields item_fields = {} for field in type_fields: # if field doesn't have properties if len(field.properties) == 0: item_fields[field.field_name] = FieldList( field_list=[FixtureItemField( # using unicode here, to cast ints, and multi-language strings field_value=unicode(di['field'][field.field_name]), properties={} )] ) else: field_list = [] field_prop_combos = di['field'][field.field_name] prop_combo_len = len(field_prop_combos) prop_dict = di[field.field_name] for x in range(0, prop_combo_len): fix_item_field = FixtureItemField( field_value=unicode(field_prop_combos[x]), properties={prop: unicode(prop_dict[prop][x]) for prop in prop_dict} ) field_list.append(fix_item_field) item_fields[field.field_name] = FieldList( field_list=field_list ) new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=item_fields, sort_key=sort_key ) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item pass old_data_item.fields = item_fields if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id: old_data_item = new_data_item messages.error(request, _("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']}) assert old_data_item.doc_type == FixtureDataItem._doc_type if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": old_data_item.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.groups for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.users for user in old_users: old_data_item.remove_user(user) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: messages.error(request, _("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name}) for raw_username in di.get('user', []): try: username = normalize_username(raw_username, domain) except ValidationError: messages.error(request, _("Invalid username: '******'. Row is not added") % {'name': raw_username}) continue user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: messages.error(request, _("Unknown user: '******'. But the row is successfully added") % {'name': raw_username}) return_val["number_of_fixtures"] = number_of_fixtures + 1 return return_val
def _run_fixture_upload(domain, workbook, replace=False, task=None): from corehq.apps.users.bulkupload import GroupMemoizer return_val = FixtureUploadResult() group_memoizer = GroupMemoizer(domain) get_location = get_memoized_location_getter(domain) data_types = [] with CouchTransaction() as transaction: type_sheets = workbook.get_all_type_sheets() total_tables = len(type_sheets) return_val.number_of_fixtures = total_tables def _update_progress(table_count, item_count, items_in_table): if task: processed = table_count * 10 + (10 * item_count / items_in_table) DownloadBase.set_progress(task, processed, 10 * total_tables) for table_number, table_def in enumerate(type_sheets): tag = table_def.table_id new_data_type = FixtureDataType( domain=domain, is_global=table_def.is_global, tag=tag, fields=table_def.fields, item_attributes=table_def.item_attributes) try: tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag) if tagged_fdt: data_type = tagged_fdt # support old usage with 'UID' elif table_def.uid: data_type = FixtureDataType.get(table_def.uid) else: data_type = new_data_type if replace and data_type != new_data_type: data_type.recursive_delete(transaction) data_type = new_data_type data_type.fields = table_def.fields data_type.item_attributes = table_def.item_attributes data_type.is_global = table_def.is_global assert data_type.doc_type == FixtureDataType._doc_type if data_type.domain != domain: data_type = new_data_type return_val.errors.append( _("'%(UID)s' is not a valid UID. But the new type is created." ) % {'UID': table_def.uid}) if table_def.delete: data_type.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): data_type = new_data_type transaction.save(data_type) data_types.append(data_type) data_items = list(workbook.get_data_sheet(data_type.tag)) items_in_table = len(data_items) for sort_key, di in enumerate(data_items): _update_progress(table_number, sort_key, items_in_table) type_fields = data_type.fields item_fields = {} for field in type_fields: # if field doesn't have properties if len(field.properties) == 0: item_fields[field.field_name] = FieldList(field_list=[ FixtureItemField( # using unicode here, to cast ints, and multi-language strings field_value=six.text_type(di['field'][ field.field_name]), properties={}) ]) else: field_list = [] field_prop_combos = di['field'][field.field_name] prop_combo_len = len(field_prop_combos) prop_dict = di[field.field_name] for x in range(0, prop_combo_len): fix_item_field = FixtureItemField( field_value=six.text_type( field_prop_combos[x]), properties={ prop: six.text_type(prop_dict[prop][x]) for prop in prop_dict }) field_list.append(fix_item_field) item_fields[field.field_name] = FieldList( field_list=field_list) item_attributes = di.get('property', {}) new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=item_fields, item_attributes=item_attributes, sort_key=sort_key) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item pass old_data_item.fields = item_fields old_data_item.item_attributes = item_attributes if old_data_item.domain != domain \ or not old_data_item.data_type_id == data_type.get_id: old_data_item = new_data_item return_val.errors.append( _("'%(UID)s' is not a valid UID. But the new item is created." ) % {'UID': di['UID']}) assert old_data_item.doc_type == FixtureDataItem._doc_type if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": old_data_item.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.groups for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.users for user in old_users: old_data_item.remove_user(user) old_locations = old_data_item.locations for location in old_locations: old_data_item.remove_location(location) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: return_val.errors.append( _("Unknown group: '%(name)s'. But the row is successfully added" ) % {'name': group_name}) for raw_username in di.get('user', []): try: username = normalize_username(str(raw_username), domain) except ValidationError: return_val.errors.append( _("Invalid username: '******'. Row is not added") % {'name': raw_username}) continue user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: return_val.errors.append( _("Unknown user: '******'. But the row is successfully added" ) % {'name': raw_username}) for name in di.get('location', []): location_cache = get_location(name) if location_cache.is_error: return_val.errors.append(location_cache.message) else: old_data_item.add_location(location_cache.location, transaction=transaction) clear_fixture_quickcache(domain, data_types) clear_fixture_cache(domain) return return_val
def get(self, fixture_id): item = FixtureDataItem.get(fixture_id) fields = {k: to_field_value(v) for k, v in item.fields.iteritems()} return self.model_class(_fixture_id=item.get_id, **fields)
def run_upload(request, domain, workbook): return_val = { "unknown_groups": [], "unknown_users": [], "number_of_fixtures": 0, } group_memoizer = GroupMemoizer(domain) data_types = workbook.get_worksheet(title='types') def _get_or_raise(container, attr): try: return container[attr] except KeyError: raise Exception("Workbook 'types' has no column '{attr}'".format(attr=attr)) number_of_fixtures = -1 with CouchTransaction() as transaction: for number_of_fixtures, dt in enumerate(data_types): tag = _get_or_raise(dt, 'tag') type_definition_fields = _get_or_raise(dt, 'field') new_data_type = FixtureDataType( domain=domain, is_global=dt.get('is_global', False), name=_get_or_raise(dt, 'name'), tag=_get_or_raise(dt, 'tag'), fields=type_definition_fields, ) try: if dt['UID']: data_type = FixtureDataType.get(dt['UID']) else: data_type = new_data_type pass data_type.fields = type_definition_fields data_type.is_global = dt.get('is_global', False) assert data_type.doc_type == FixtureDataType._doc_type if data_type.domain != domain: data_type = new_data_type messages.error(request, _("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': dt['UID']}) if dt[DELETE_HEADER] == "Y" or dt[DELETE_HEADER] == "y": data_type.recursive_delete(transaction) continue except (ResourceNotFound, KeyError) as e: data_type = new_data_type transaction.save(data_type) data_items = workbook.get_worksheet(data_type.tag) for sort_key, di in enumerate(data_items): # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH item_fields = di['field'] for field in type_definition_fields: if not item_fields.has_key(field): raise Exception(_("Workbook '%(tag)s' does not contain the column " + "'%(field)s' specified in its 'types' definition") % {'tag': tag, 'field': field}) item_fields_list = di['field'].keys() for field in item_fields_list: if not field in type_definition_fields: raise Exception(_("""Workbook '%(tag)s' has an extra column '%(field)s' that's not defined in its 'types' definition""") % {'tag': tag, 'field': field}) new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=item_fields, sort_key=sort_key ) try: if di['UID']: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item pass old_data_item.fields = di['field'] if old_data_item.domain != domain: old_data_item = new_data_item messages.error(request, _("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID'] }) assert old_data_item.doc_type == FixtureDataItem._doc_type assert old_data_item.data_type_id == data_type.get_id if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": old_data_item.recursive_delete(transaction) continue except (ResourceNotFound, KeyError) as e: old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.get_groups() for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.get_users() for user in old_users: old_data_item.remove_user(user) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: messages.error(request, _("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name}) for raw_username in di.get('user', []): try: username = normalize_username(raw_username, domain) except ValidationError: messages.error(request, _("Invalid username: '******'. Row is not added") % {'name': raw_username}) continue user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: messages.error(request, _("Unknown user: '******'. But the row is successfully added") % {'name': raw_username}) return_val["number_of_fixtures"] = number_of_fixtures + 1 return return_val
def run_upload(domain, workbook, replace=False, task=None): return_val = FixtureUploadResult() group_memoizer = GroupMemoizer(domain) get_location = get_memoized_location(domain) def diff_lists(list_a, list_b): set_a = set(list_a) set_b = set(list_b) not_in_b = set_a.difference(set_b) not_in_a = set_a.difference(set_a) return list(not_in_a), list(not_in_b) with CouchTransaction() as transaction: type_sheets = workbook.get_all_type_sheets() total_tables = len(type_sheets) return_val.number_of_fixtures = total_tables def _update_progress(table_count, item_count, items_in_table): if task: processed = table_count * 10 + (10. * item_count / items_in_table) DownloadBase.set_progress(task, processed, 10 * total_tables) for table_number, table_def in enumerate(type_sheets): tag = table_def.table_id new_data_type = FixtureDataType( domain=domain, is_global=table_def.is_global, tag=table_def.table_id, fields=table_def.fields, item_attributes=table_def.item_attributes) try: tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag) if tagged_fdt: data_type = tagged_fdt # support old usage with 'UID' elif table_def.uid: data_type = FixtureDataType.get(table_def.uid) else: data_type = new_data_type if replace and data_type != new_data_type: data_type.recursive_delete(transaction) data_type = new_data_type data_type.fields = table_def.fields data_type.item_attributes = table_def.item_attributes data_type.is_global = table_def.is_global assert data_type.doc_type == FixtureDataType._doc_type if data_type.domain != domain: data_type = new_data_type return_val.errors.append( _("'%(UID)s' is not a valid UID. But the new type is created." ) % {'UID': table_def.uid}) if table_def.delete: data_type.recursive_delete(transaction) continue except (ResourceNotFound, KeyError) as e: data_type = new_data_type transaction.save(data_type) data_items = list(workbook.get_data_sheet(data_type)) items_in_table = len(data_items) for sort_key, di in enumerate(data_items): _update_progress(table_number, sort_key, items_in_table) # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH item_fields_list = di['field'].keys() if 'field' in di else [] not_in_sheet, not_in_types = diff_lists( item_fields_list, data_type.fields_without_attributes) if len(not_in_sheet) > 0: error_message = _( FAILURE_MESSAGES["has_no_field_column"]).format( tag=tag, field=not_in_sheet[0]) raise ExcelMalformatException(error_message) if len(not_in_types) > 0: error_message = _( FAILURE_MESSAGES["has_extra_column"]).format( tag=tag, field=not_in_types[0]) raise ExcelMalformatException(error_message) # check that this item has all the properties listed in its 'types' definition item_attributes_list = di['property'].keys( ) if 'property' in di else [] not_in_sheet, not_in_types = diff_lists( item_attributes_list, data_type.item_attributes) if len(not_in_sheet) > 0: error_message = _( FAILURE_MESSAGES["has_no_field_column"]).format( tag=tag, field=not_in_sheet[0]) raise ExcelMalformatException(error_message) if len(not_in_types) > 0: error_message = _( FAILURE_MESSAGES["has_extra_column"]).format( tag=tag, field=not_in_types[0]) raise ExcelMalformatException(error_message) # check that properties in 'types' sheet vs item-sheet MATCH for field in data_type.fields: if len(field.properties) > 0: sheet_props = di.get(field.field_name, {}) sheet_props_list = sheet_props.keys() type_props = field.properties not_in_sheet, not_in_types = diff_lists( sheet_props_list, type_props) if len(not_in_sheet) > 0: error_message = _( FAILURE_MESSAGES["sheet_has_no_property"] ).format(tag=tag, property=not_in_sheet[0], field=field.field_name) raise ExcelMalformatException(error_message) if len(not_in_types) > 0: error_message = _( FAILURE_MESSAGES["sheet_has_extra_property"] ).format(tag=tag, property=not_in_types[0], field=field.field_name) raise ExcelMalformatException(error_message) # check that fields with properties are numbered if type(di['field'][field.field_name]) != list: error_message = _( FAILURE_MESSAGES["invalid_field_with_property"] ).format(field=field.field_name) raise ExcelMalformatException(error_message) field_prop_len = len(di['field'][field.field_name]) for prop in sheet_props: if type(sheet_props[prop]) != list: error_message = _( FAILURE_MESSAGES["invalid_property"] ).format(field=field.field_name, prop=prop) raise ExcelMalformatException(error_message) if len(sheet_props[prop]) != field_prop_len: error_message = _(FAILURE_MESSAGES[ "wrong_field_property_combos"]).format( field=field.field_name, prop=prop) raise ExcelMalformatException(error_message) # excel format check should have been covered by this line. Can make assumptions about data now type_fields = data_type.fields item_fields = {} for field in type_fields: # if field doesn't have properties if len(field.properties) == 0: item_fields[field.field_name] = FieldList(field_list=[ FixtureItemField( # using unicode here, to cast ints, and multi-language strings field_value=unicode(di['field'][ field.field_name]), properties={}) ]) else: field_list = [] field_prop_combos = di['field'][field.field_name] prop_combo_len = len(field_prop_combos) prop_dict = di[field.field_name] for x in range(0, prop_combo_len): fix_item_field = FixtureItemField( field_value=unicode(field_prop_combos[x]), properties={ prop: unicode(prop_dict[prop][x]) for prop in prop_dict }) field_list.append(fix_item_field) item_fields[field.field_name] = FieldList( field_list=field_list) item_attributes = di.get('property', {}) new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=item_fields, item_attributes=item_attributes, sort_key=sort_key) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item pass old_data_item.fields = item_fields old_data_item.item_attributes = item_attributes if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id: old_data_item = new_data_item return_val.errors.append( _("'%(UID)s' is not a valid UID. But the new item is created." ) % {'UID': di['UID']}) assert old_data_item.doc_type == FixtureDataItem._doc_type if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": old_data_item.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.groups for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.users for user in old_users: old_data_item.remove_user(user) old_locations = old_data_item.locations for location in old_locations: old_data_item.remove_location(location) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: return_val.errors.append( _("Unknown group: '%(name)s'. But the row is successfully added" ) % {'name': group_name}) for raw_username in di.get('user', []): try: username = normalize_username(str(raw_username), domain) except ValidationError: return_val.errors.append( _("Invalid username: '******'. Row is not added") % {'name': raw_username}) continue user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: return_val.errors.append( _("Unknown user: '******'. But the row is successfully added" ) % {'name': raw_username}) for name in di.get('location', []): location_cache = get_location(name) if location_cache.is_error: return_val.errors.append(location_cache.message) else: old_data_item.add_location(location_cache.location, transaction=transaction) return return_val
def run_upload(domain, workbook, replace=False, task=None): return_val = FixtureUploadResult() group_memoizer = GroupMemoizer(domain) pre_populate_location_groups(group_memoizer, domain) def diff_lists(list_a, list_b): set_a = set(list_a) set_b = set(list_b) not_in_b = set_a.difference(set_b) not_in_a = set_a.difference(set_a) return list(not_in_a), list(not_in_b) with CouchTransaction() as transaction: type_sheets = workbook.get_all_type_sheets() total_tables = len(type_sheets) return_val.number_of_fixtures = total_tables def _update_progress(table_count, item_count, items_in_table): if task: processed = table_count * 10 + (10. * item_count / items_in_table) DownloadBase.set_progress(task, processed, 10 * total_tables) for table_number, table_def in enumerate(type_sheets): tag = table_def.table_id new_data_type = FixtureDataType( domain=domain, is_global=table_def.is_global, tag=table_def.table_id, fields=table_def.fields, item_attributes=table_def.item_attributes ) try: tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag) if tagged_fdt: data_type = tagged_fdt # support old usage with 'UID' elif table_def.uid: data_type = FixtureDataType.get(table_def.uid) else: data_type = new_data_type if replace and data_type != new_data_type: data_type.recursive_delete(transaction) data_type = new_data_type data_type.fields = table_def.fields data_type.item_attributes = table_def.item_attributes data_type.is_global = table_def.is_global assert data_type.doc_type == FixtureDataType._doc_type if data_type.domain != domain: data_type = new_data_type return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': table_def.uid}) if table_def.delete: data_type.recursive_delete(transaction) continue except (ResourceNotFound, KeyError) as e: data_type = new_data_type transaction.save(data_type) data_items = list(workbook.get_data_sheet(data_type)) items_in_table = len(data_items) for sort_key, di in enumerate(data_items): _update_progress(table_number, sort_key, items_in_table) # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH item_fields_list = di['field'].keys() if 'field' in di else [] not_in_sheet, not_in_types = diff_lists(item_fields_list, data_type.fields_without_attributes) if len(not_in_sheet) > 0: error_message = _(FAILURE_MESSAGES["has_no_field_column"]).format(tag=tag, field=not_in_sheet[0]) raise ExcelMalformatException(error_message) if len(not_in_types) > 0: error_message = _(FAILURE_MESSAGES["has_extra_column"]).format(tag=tag, field=not_in_types[0]) raise ExcelMalformatException(error_message) # check that this item has all the properties listed in its 'types' definition item_attributes_list = di['property'].keys() if 'property' in di else [] not_in_sheet, not_in_types = diff_lists(item_attributes_list, data_type.item_attributes) if len(not_in_sheet) > 0: error_message = _(FAILURE_MESSAGES["has_no_field_column"]).format(tag=tag, field=not_in_sheet[0]) raise ExcelMalformatException(error_message) if len(not_in_types) > 0: error_message = _(FAILURE_MESSAGES["has_extra_column"]).format(tag=tag, field=not_in_types[0]) raise ExcelMalformatException(error_message) # check that properties in 'types' sheet vs item-sheet MATCH for field in data_type.fields: if len(field.properties) > 0: sheet_props = di.get(field.field_name, {}) sheet_props_list = sheet_props.keys() type_props = field.properties not_in_sheet, not_in_types = diff_lists(sheet_props_list, type_props) if len(not_in_sheet) > 0: error_message = _(FAILURE_MESSAGES["sheet_has_no_property"]).format( tag=tag, property=not_in_sheet[0], field=field.field_name ) raise ExcelMalformatException(error_message) if len(not_in_types) > 0: error_message = _(FAILURE_MESSAGES["sheet_has_extra_property"]).format( tag=tag, property=not_in_types[0], field=field.field_name ) raise ExcelMalformatException(error_message) # check that fields with properties are numbered if type(di['field'][field.field_name]) != list: error_message = _(FAILURE_MESSAGES["invalid_field_with_property"]).format(field=field.field_name) raise ExcelMalformatException(error_message) field_prop_len = len(di['field'][field.field_name]) for prop in sheet_props: if type(sheet_props[prop]) != list: error_message = _(FAILURE_MESSAGES["invalid_property"]).format( field=field.field_name, prop=prop ) raise ExcelMalformatException(error_message) if len(sheet_props[prop]) != field_prop_len: error_message = _(FAILURE_MESSAGES["wrong_field_property_combos"]).format( field=field.field_name, prop=prop ) raise ExcelMalformatException(error_message) # excel format check should have been covered by this line. Can make assumptions about data now type_fields = data_type.fields item_fields = {} for field in type_fields: # if field doesn't have properties if len(field.properties) == 0: item_fields[field.field_name] = FieldList( field_list=[FixtureItemField( # using unicode here, to cast ints, and multi-language strings field_value=unicode(di['field'][field.field_name]), properties={} )] ) else: field_list = [] field_prop_combos = di['field'][field.field_name] prop_combo_len = len(field_prop_combos) prop_dict = di[field.field_name] for x in range(0, prop_combo_len): fix_item_field = FixtureItemField( field_value=unicode(field_prop_combos[x]), properties={prop: unicode(prop_dict[prop][x]) for prop in prop_dict} ) field_list.append(fix_item_field) item_fields[field.field_name] = FieldList( field_list=field_list ) item_attributes = di.get('property', {}) new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=item_fields, item_attributes=item_attributes, sort_key=sort_key ) try: if di['UID'] and not replace: old_data_item = FixtureDataItem.get(di['UID']) else: old_data_item = new_data_item pass old_data_item.fields = item_fields old_data_item.item_attributes = item_attributes if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id: old_data_item = new_data_item return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']}) assert old_data_item.doc_type == FixtureDataItem._doc_type if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y": old_data_item.recursive_delete(transaction) continue except (ResourceNotFound, KeyError): old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.groups for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.users for user in old_users: old_data_item.remove_user(user) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: return_val.errors.append(_("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name}) for raw_username in di.get('user', []): try: username = normalize_username(str(raw_username), domain) except ValidationError: return_val.errors.append(_("Invalid username: '******'. Row is not added") % {'name': raw_username}) continue user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: return_val.errors.append(_("Unknown user: '******'. But the row is successfully added") % {'name': raw_username}) return return_val
def run_upload_api(request, domain, workbook): return_val = { "unknown_groups": [], "unknown_users": [], "number_of_fixtures": 0, } group_memoizer = GroupMemoizer(domain) data_types = workbook.get_worksheet(title='types') def _get_or_raise(container, attr): try: return container[attr] except KeyError: raise Exception("Workbook 'types' has no column '{attr}'".format(attr=attr)) number_of_fixtures = -1 with CouchTransaction() as transaction: for number_of_fixtures, dt in enumerate(data_types): tag = _get_or_raise(dt, 'tag') data_type_results = FixtureDataType.by_domain_tag(domain, tag) if len(data_type_results) == 0: data_type = FixtureDataType( domain=domain, name=_get_or_raise(dt, 'name'), tag=_get_or_raise(dt, 'tag'), fields=_get_or_raise(dt, 'field'), ) transaction.save(data_type) else: for x in data_type_results: data_type = x data_items = workbook.get_worksheet(data_type.tag) for sort_key, di in enumerate(data_items): new_data_item = FixtureDataItem( domain=domain, data_type_id=data_type.get_id, fields=di['field'], sort_key=sort_key ) try: old_data_item = FixtureDataItem.get(di['UID']) assert old_data_item.domain == domain assert old_data_item.doc_type == FixtureDataItem._doc_type assert old_data_item.data_type_id == data_type.get_id if di.get(DELETE_HEADER) in ("Y", "y"): old_data_item.recursive_delete(transaction) continue old_data_item.fields = di['field'] transaction.save(old_data_item) except (AttributeError, KeyError, ResourceNotFound, AssertionError): old_data_item = new_data_item transaction.save(old_data_item) old_groups = old_data_item.get_groups() for group in old_groups: old_data_item.remove_group(group) old_users = old_data_item.get_users() for user in old_users: old_data_item.remove_user(user) for group_name in di.get('group', []): group = group_memoizer.by_name(group_name) if group: old_data_item.add_group(group, transaction=transaction) else: messages.error(request, "Unknown group: %s" % group_name) for raw_username in di.get('user', []): username = normalize_username(raw_username, domain) user = CommCareUser.get_by_username(username) if user: old_data_item.add_user(user) else: messages.error(request, "Unknown user: %s" % raw_username) return_val["number_of_fixtures"] = number_of_fixtures + 1 return return_val
def delete(self): if self._fixture_id is None: return item = FixtureDataItem.get(self._fixture_id) item.delete()