def iter_fixture_items_for_data_type(domain, data_type_id): from corehq.apps.fixtures.models import FixtureDataItem for row in paginate_view(FixtureDataItem.get_db(), 'fixtures/data_items_by_domain_type', chunk_size=1000, startkey=[domain, data_type_id], endkey=[domain, data_type_id, {}], reduce=False, include_docs=True): yield FixtureDataItem.wrap(row['doc'])
def _import_fixtures(domain): for fixture_name, filename in [ ('recipes', 'recipes.csv'), ('conv_factors', 'conv_factors.csv'), ('food_list', 'food_list.csv'), ('food_composition_table', 'food_composition_table.csv'), ]: fields, rows = _read_csv(filename) data_type = FixtureDataType( domain=domain, tag=fixture_name, fields=[FixtureTypeField(field_name=field) for field in fields], ) data_type.save() with IterDB(FixtureDataItem.get_db(), chunksize=1000) as iter_db: for vals in rows: fixture_data_item = _mk_fixture_data_item( domain, data_type._id, fields, vals) iter_db.save(fixture_data_item)
def _run_fast_fixture_upload(domain, workbook, task=None): """This upload should be much faster than the default _run_fixture_upload with the following trade-offs: * Does not support any fixture ownership. All fixtures must be global * Manually creates the JSON documents instead of using models * Delays all fixture item deletes to an asynchronous task * Creates tables one by one instead of attempting an "all or nothing" approach """ return_val = FixtureUploadResult() type_sheets = workbook.get_all_type_sheets() for table_definition in type_sheets: if not table_definition.is_global: return_val.errors.append( _("type {lookup_table_name} is not defined as global").format( lookup_table_name=table_definition.table_id ) ) return return_val total_tables = len(type_sheets) return_val.number_of_fixtures = total_tables def _update_progress(table_count, item_count, items_in_table): if task: processed = table_count * 10 + (10 * item_count / items_in_table) DownloadBase.set_progress(task, processed, 10 * total_tables) existing_data_types_by_tag = { data_type.tag: data_type for data_type in FixtureDataType.by_domain(domain) } for table_number, table_def in enumerate(type_sheets): data_type = { "_id": uuid.uuid4().hex, "doc_type": "FixtureDataType", "domain": domain, "is_global": True, "description": None, "fields": [field.to_json() for field in table_def.fields], "copy_from": None, "tag": table_def.table_id, "item_attributes": table_def.item_attributes } data_item_docs_to_save = [] data_items = list(workbook.get_data_sheet(data_type['tag'])) items_in_table = len(data_items) for sort_key, di in enumerate(data_items): _update_progress(table_number, sort_key, items_in_table) type_fields = table_def.fields item_fields = { field.field_name: _process_item_field(field, di).to_json() for field in type_fields } item_attributes = di.get('property', {}) data_item = { "_id": uuid.uuid4().hex, "doc_type": "FixtureDataItem", "domain": domain, "sort_key": sort_key, "fields": item_fields, "data_type_id": data_type['_id'], "item_attributes": item_attributes } data_item_docs_to_save.append(data_item) # save all the data items in the fixture, before the data type # This ensure that all data items are created before the data type is created # which could result in partial table upload try: for docs in chunked(data_item_docs_to_save, 1000): FixtureDataItem.get_db().save_docs(docs) except (BulkSaveError, HTTPError): return_val.errors.append( _("Error occurred while creating {lookup_table_name}. This table was not created").format( lookup_table_name=data_type['tag'] ) ) continue data_type_docs = [data_type] existing_data_type = existing_data_types_by_tag.get(data_type['tag']) if existing_data_type: # delete the old data type in the same request data_type_docs.append({ "_id": existing_data_type._id, "_rev": existing_data_type._rev, "_deleted": True }) # the following save_docs can result in two issues: # * the delete fails, new doc save succeeds meaning that there are two data types with the same tag # * the delete succeeds, new doc save fails meaning that there is no data type with the desired tag try: FixtureDataType.get_db().save_docs(data_type_docs) except (BulkSaveError, HTTPError): return_val.errors.append( _("Error occurred while creating {lookup_table_name}. This table was not created").format( lookup_table_name=data_type['tag'] ) ) continue if existing_data_type: return_val.messages.append( _("Pre-existing definition of {lookup_table_name} deleted").format( lookup_table_name=existing_data_type.tag ) ) return_val.messages.append( _("Table {lookup_table_name} successfully uploaded").format(lookup_table_name=data_type['tag']), ) if existing_data_type: from corehq.apps.fixtures.tasks import delete_unneeded_fixture_data_item # delay removing data items for the previously delete type as that requires a # couch view hit which introduces another opportunity for failure delete_unneeded_fixture_data_item.delay(domain, existing_data_type._id) clear_fixture_quickcache(domain, [existing_data_type]) clear_fixture_cache(domain) return return_val
def delete_fixture_items_for_data_type(domain, data_type_id): from corehq.apps.fixtures.models import FixtureDataItem iter_bulk_delete(FixtureDataItem.get_db(), [ i["_id"] for i in iter_fixture_items_for_data_type(domain, data_type_id) ])