def _import_cases(domain, user): BULK_UPLOAD_DATE_OPENED.set(domain, True, NAMESPACE_DOMAIN) _import_case_type(domain, FOODRECALL_CASE_TYPE, 'foodrecall_cases.csv', user) _import_case_type(domain, FOOD_CASE_TYPE, 'food_cases.csv', user) BULK_UPLOAD_DATE_OPENED.set(domain, False, NAMESPACE_DOMAIN) _update_case_id_properties(domain, user)
def get_special_fields(domain=None): special_fields = [ FieldSpec( field='name', description=_("This field will be used to set the case's name."), show_in_menu=True), FieldSpec( field='owner_name', description=_( "This field will assign the case to a new owner given by " "Username, Group name, or Organization name."), ), FieldSpec( field='owner_id', description=_( "This field will assign the case to a new owner given by " "User ID, Group ID, or Organization ID.")), FieldSpec(field='external_id', description=_("This field will set the case's external_id")), FieldSpec(field='parent_external_id', description=_( "This field will assign the case a new parent given by " "the parent case's external_id. " "You must use along with parent_type.")), FieldSpec(field='parent_id', description=_( "This field will assign the case a new parent given by " "the parent's Case ID. " "You must use along with parent_type.")), FieldSpec(field='parent_type', description=_( "Use to specify the parent's case type. " "Usually used with parent_id or parent_external_id")), FieldSpec(field='parent_relationship_type', description=_( "Whether the relationship with the parent is 'child' or " "'extension'. Default value is 'child'" "Values other than 'child' or 'extension' are invalid. " "Used with parent_id or parent_external_id columns")), FieldSpec( field='parent_identifier', description=_( "The index identifier when creating child/extension cases " "Used with parent_id or parent_external_id columns")), FieldSpec(field='close', description=_( "This field will be used to close cases. " "Any case with 'yes' in this column will be closed.")), ] if domain and BULK_UPLOAD_DATE_OPENED.enabled(domain): special_fields.append( FieldSpec( field='date_opened', description=_( "The date opened property for this case will be changed. " "Please do not use unless you know what you are doing"))) return special_fields
def get_special_fields(domain=None): special_fields = [ FieldSpec( field='name', description=_("This field will be used to set the case's name."), show_in_menu=True), FieldSpec( field='owner_name', description=_("This field will assign the case to a new owner given by " "Username, Group name, or Organization name."), ), FieldSpec( field='owner_id', description=_("This field will assign the case to a new owner given by " "User ID, Group ID, or Organization ID.")), FieldSpec( field='external_id', description=_("This field will set the case's external_id")), FieldSpec( field='parent_external_id', description=_("This field will assign the case a new parent given by " "the parent case's external_id. " "You must use along with parent_type.")), FieldSpec( field='parent_id', description=_("This field will assign the case a new parent given by " "the parent's Case ID. " "You must use along with parent_type.")), FieldSpec( field='parent_type', description=_("Use to specify the parent's case type. " "Usually used with parent_id or parent_external_id")), FieldSpec( field='parent_ref', description=_("This is a deprecated feature needed for a handful of clients. " "Please do not use unless you know what you are doing"), discoverable=False), FieldSpec( field='close', description=_("This field will be used to close cases. " "Any case with 'yes' in this column will be closed.")), ] if domain and BULK_UPLOAD_DATE_OPENED.enabled(domain): special_fields.append( FieldSpec( field='date_opened', description=_( "The date opened property for this case will be changed. " "Please do not use unless you know what you are doing" ) ) ) return special_fields
parent_ref: (parent_case.type, parent_id) } except ResourceNotFound: errors.add(ImportErrors.InvalidParentId, i + 1, 'parent_id') continue elif parent_external_id: parent_case, error = importer_util.lookup_case( 'external_id', parent_external_id, domain, parent_type) if parent_case: extras['index'] = { parent_ref: (parent_type, parent_case.case_id) } case_name = fields_to_update.pop('name', None) if BULK_UPLOAD_DATE_OPENED.enabled(domain): date_opened = fields_to_update.pop(CASE_TAG_DATE_OPENED, None) if date_opened: extras['date_opened'] = date_opened if not case: id = uuid.uuid4().hex if config.search_field == 'external_id': extras['external_id'] = search_id elif external_id: extras['external_id'] = external_id try: caseblock = CaseBlock(create=True, case_id=id,
def _get_date_opened(self): if self.date_opened and BULK_UPLOAD_DATE_OPENED.enabled(self.domain): return self.date_opened
def do_import(spreadsheet, config, domain, task=None, chunksize=CASEBLOCK_CHUNKSIZE, record_form_callback=None): match_count = created_count = too_many_matches = num_chunks = 0 errors = importer_util.ImportErrorDetail() user = CouchUser.get_by_user_id(config.couch_user_id, domain) username = user.username user_id = user._id # keep a cache of id lookup successes to help performance id_cache = {} name_cache = {} caseblocks = [] ids_seen = set() track_load = case_load_counter("case_importer", domain) def _submit_caseblocks(domain, case_type, caseblocks): err = False if caseblocks: try: form, cases = submit_case_blocks( [cb.case.as_string().decode('utf-8') for cb in caseblocks], domain, username, user_id, device_id=__name__ + ".do_import", ) if form.is_error: errors.add( error=ImportErrors.ImportErrorMessage, row_number=form.problem ) except Exception: err = True for row_number, case in caseblocks: errors.add( error=ImportErrors.ImportErrorMessage, row_number=row_number ) else: if record_form_callback: record_form_callback(form.form_id) properties = set().union(*[set(c.dynamic_case_properties().keys()) for c in cases]) if case_type and len(properties): add_inferred_export_properties.delay( 'CaseImporter', domain, case_type, properties, ) else: _soft_assert = soft_assert(notify_admins=True) _soft_assert( len(properties) == 0, 'error adding inferred export properties in domain ' '({}): {}'.format(domain, ", ".join(properties)) ) return err row_count = spreadsheet.max_row for i, row in enumerate(spreadsheet.iter_row_dicts()): if task: set_task_progress(task, i, row_count) # skip first row (header row) if i == 0: continue search_id = importer_util.parse_search_id(config, row) fields_to_update = importer_util.populate_updated_fields(config, row) if not any(fields_to_update.values()): # if the row was blank, just skip it, no errors continue if config.search_field == 'external_id' and not search_id: # do not allow blank external id since we save this errors.add(ImportErrors.BlankExternalId, i + 1) continue external_id = fields_to_update.pop('external_id', None) parent_id = fields_to_update.pop('parent_id', None) parent_external_id = fields_to_update.pop('parent_external_id', None) parent_type = fields_to_update.pop('parent_type', config.case_type) parent_ref = fields_to_update.pop('parent_ref', 'parent') to_close = fields_to_update.pop('close', False) if any([lookup_id and lookup_id in ids_seen for lookup_id in [search_id, parent_id, parent_external_id]]): # clear out the queue to make sure we've processed any potential # cases we want to look up # note: these three lines are repeated a few places, and could be converted # to a function that makes use of closures (and globals) to do the same thing, # but that seems sketchier than just beeing a little RY _submit_caseblocks(domain, config.case_type, caseblocks) num_chunks += 1 caseblocks = [] ids_seen = set() # also clear ids_seen, since all the cases will now be in the database case, error = importer_util.lookup_case( config.search_field, search_id, domain, config.case_type ) track_load() if case: if case.type != config.case_type: continue elif error == LookupErrors.NotFound: if not config.create_new_cases: continue elif error == LookupErrors.MultipleResults: too_many_matches += 1 continue uploaded_owner_name = fields_to_update.pop('owner_name', None) uploaded_owner_id = fields_to_update.pop('owner_id', None) if uploaded_owner_name: # If an owner name was provided, replace the provided # uploaded_owner_id with the id of the provided group or owner try: uploaded_owner_id = importer_util.get_id_from_name(uploaded_owner_name, domain, name_cache) except SQLLocation.MultipleObjectsReturned: errors.add(ImportErrors.DuplicateLocationName, i + 1) continue if not uploaded_owner_id: errors.add(ImportErrors.InvalidOwnerName, i + 1, 'owner_name') continue if uploaded_owner_id: # If an owner_id mapping exists, verify it is a valid user # or case sharing group if importer_util.is_valid_id(uploaded_owner_id, domain, id_cache): owner_id = uploaded_owner_id id_cache[uploaded_owner_id] = True else: errors.add(ImportErrors.InvalidOwnerId, i + 1, 'owner_id') id_cache[uploaded_owner_id] = False continue else: # if they didn't supply an owner_id mapping, default to current # user owner_id = user_id extras = {} if parent_id: try: parent_case = CaseAccessors(domain).get_case(parent_id) track_load() if parent_case.domain == domain: extras['index'] = { parent_ref: (parent_case.type, parent_id) } except ResourceNotFound: errors.add(ImportErrors.InvalidParentId, i + 1, 'parent_id') continue elif parent_external_id: parent_case, error = importer_util.lookup_case( 'external_id', parent_external_id, domain, parent_type ) track_load() if parent_case: extras['index'] = { parent_ref: (parent_type, parent_case.case_id) } case_name = fields_to_update.pop('name', None) if BULK_UPLOAD_DATE_OPENED.enabled(domain): date_opened = fields_to_update.pop(CASE_TAG_DATE_OPENED, None) if date_opened: extras['date_opened'] = date_opened if not case: id = uuid.uuid4().hex if config.search_field == 'external_id': extras['external_id'] = search_id elif external_id: extras['external_id'] = external_id try: caseblock = CaseBlock( create=True, case_id=id, owner_id=owner_id, user_id=user_id, case_type=config.case_type, case_name=case_name or '', update=fields_to_update, **extras ) caseblocks.append(RowAndCase(i, caseblock)) created_count += 1 if external_id: ids_seen.add(external_id) except CaseBlockError: errors.add(ImportErrors.CaseGeneration, i + 1) else: if external_id: extras['external_id'] = external_id if uploaded_owner_id: extras['owner_id'] = owner_id if to_close == 'yes': extras['close'] = True if case_name is not None: extras['case_name'] = case_name try: caseblock = CaseBlock( create=False, case_id=case.case_id, update=fields_to_update, **extras ) caseblocks.append(RowAndCase(i, caseblock)) match_count += 1 except CaseBlockError: errors.add(ImportErrors.CaseGeneration, i + 1) # check if we've reached a reasonable chunksize # and if so submit if len(caseblocks) >= chunksize: _submit_caseblocks(domain, config.case_type, caseblocks) num_chunks += 1 caseblocks = [] # final purge of anything left in the queue if _submit_caseblocks(domain, config.case_type, caseblocks): match_count -= 1 num_chunks += 1 return { 'created_count': created_count, 'match_count': match_count, 'too_many_matches': too_many_matches, 'errors': errors.as_dict(), 'num_chunks': num_chunks, }