def make_and_get_indent(indent, valid_user, depth): data = { "indented_goods_nomenclature": { "sid": indent.indented_goods_nomenclature.sid, "item_id": indent.indented_goods_nomenclature.item_id, "suffix": indent.indented_goods_nomenclature.suffix, }, "taric_template": "taric/goods_nomenclature_indent.xml", "update_type": indent.update_type, "sid": indent.sid, "start_date": f"{indent.valid_between.lower:%Y-%m-%d}", "indent": depth, } xml = generate_test_import_xml(data) process_taric_xml_stream( xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED.value, ) return models.GoodsNomenclatureIndent.objects.filter( sid=indent.sid, ).last()
def test_goods_nomenclature_successor_importer_create(valid_user, date_ranges): good = factories.GoodsNomenclatureFactory( update_type=UpdateType.CREATE.value, valid_between=date_ranges.normal, ) successor = factories.GoodsNomenclatureFactory( update_type=UpdateType.CREATE.value, valid_between=date_ranges.adjacent_later, ) successor_link = factories.GoodsNomenclatureSuccessorFactory.build( replaced_goods_nomenclature=good, absorbed_into_goods_nomenclature=successor, update_type=UpdateType.CREATE.value, ) xml = generate_test_import_xml( serializers.GoodsNomenclatureSuccessorSerializer( successor_link, context={"format": "xml"}, ).data, ) process_taric_xml_stream( xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED.value, ) db_link = models.GoodsNomenclatureSuccessor.objects.get( replaced_goods_nomenclature__sid=good.sid, ) assert db_link db_good = models.GoodsNomenclature.objects.get(sid=good.sid) assert db_good.successors.get() == successor
def run_import(xml, workflow_status=WorkflowStatus.PUBLISHED, record_group=None): process_taric_xml_stream( xml, workbasket_id=None, workbasket_status=workflow_status, partition_scheme=get_partition_scheme(), username=valid_user.username, record_group=record_group, )
def import_chunk( chunk_pk: int, workbasket_id: str, workbasket_status: str, partition_scheme_setting: str, username: str, record_group: Sequence[str] = None, ): """ Task for importing an XML chunk into the database. This task must ensure the chunks workbasket_status reflects the import process, whether it is currently running, errored or done. Once complete it is also responsible for finding and setting up the next chunk tasks. """ partition_scheme = get_partition_scheme(partition_scheme_setting) chunk = models.ImporterXMLChunk.objects.get(pk=chunk_pk) logger.info( "RUNNING CHUNK Batch: %s Record code: %s Chapter heading: %s Chunk number: %d", chunk.batch.name, chunk.record_code, chunk.chapter, chunk.chunk_number, ) chunk.status = models.ImporterChunkStatus.RUNNING chunk.save() try: process_taric_xml_stream( BytesIO(chunk.chunk_text.encode()), workbasket_id, workbasket_status, partition_scheme, username, record_group=record_group, ) except Exception as e: chunk.status = models.ImporterChunkStatus.ERRORED chunk.save() raise e chunk.status = models.ImporterChunkStatus.DONE chunk.save() find_and_run_next_batch_chunks( chunk.batch, workbasket_id, workbasket_status, partition_scheme_setting, username, )
def test_goods_nomenclature_successor_importer_delete(valid_user, date_ranges): good = factories.GoodsNomenclatureFactory( update_type=UpdateType.CREATE.value, valid_between=date_ranges.normal, ) successor = factories.GoodsNomenclatureFactory( update_type=UpdateType.CREATE.value, valid_between=date_ranges.adjacent_later, ) factories.GoodsNomenclatureSuccessorFactory( replaced_goods_nomenclature=good, absorbed_into_goods_nomenclature=successor, update_type=UpdateType.CREATE.value, ) updated_good = factories.GoodsNomenclatureFactory( sid=good.sid, item_id=good.item_id, suffix=good.suffix, version_group=good.version_group, update_type=UpdateType.UPDATE.value, valid_between=date_ranges.no_end, ) successor_link = factories.GoodsNomenclatureSuccessorFactory.build( replaced_goods_nomenclature=updated_good, absorbed_into_goods_nomenclature=successor, update_type=UpdateType.DELETE.value, ) xml = generate_test_import_xml( serializers.GoodsNomenclatureSuccessorSerializer( successor_link, context={"format": "xml"}, ).data, ) process_taric_xml_stream( xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED.value, ) db_link = models.GoodsNomenclatureSuccessor.objects.filter( replaced_goods_nomenclature__sid=good.sid, ) assert not db_link.latest_approved().exists() assert db_link.latest_deleted().exists()
def test_replacement_importer_update(valid_user): replacement = factories.ReplacementFactory.create() data = { "enacting_regulation": { "role_type": replacement.enacting_regulation.role_type, "regulation_id": replacement.enacting_regulation.regulation_id, }, "target_regulation": { "role_type": replacement.target_regulation.role_type, "regulation_id": replacement.target_regulation.regulation_id, }, "taric_template": "taric/replacement.xml", "update_type": UpdateType.UPDATE.value, "measure_type_id": replacement.measure_type_id, "geographical_area_id": "UK", "chapter_heading": replacement.chapter_heading, } xml = generate_test_import_xml(data) process_taric_xml_stream(xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED.value) replacements = models.Replacement.objects.filter( enacting_regulation__regulation_id=replacement.enacting_regulation. regulation_id, target_regulation__regulation_id=replacement.target_regulation. regulation_id, ) assert replacements.count() == 2 replacement = replacements.get(update_type=UpdateType.UPDATE) assert replacement.measure_type_id == replacement.measure_type_id assert replacement.geographical_area_id == replacement.geographical_area_id assert replacement.chapter_heading == replacement.chapter_heading assert replacement.enacting_regulation == replacement.enacting_regulation assert replacement.target_regulation == replacement.target_regulation version_group = replacement.version_group assert version_group.versions.count() == 2 assert (version_group == replacements.get( update_type=UpdateType.CREATE).version_group) assert version_group.current_version == replacement
def check( model: Union[TrackedModel, Type[DjangoModelFactory]], serializer: Type[TrackedModelSerializer], ) -> TrackedModel: get_nursery().cache.clear() settings.SKIP_WORKBASKET_VALIDATION = True if isinstance(model, type) and issubclass(model, DjangoModelFactory): model = model.build(update_type=UpdateType.CREATE) assert isinstance( model, TrackedModel, ), "Either a factory or an object instance needs to be provided" xml = generate_test_import_xml( serializer(model, context={"format": "xml"}).data, ) process_taric_xml_stream( xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED, ) db_kwargs = model.get_identifying_fields() imported = model.__class__.objects.get_latest_version(**db_kwargs) checked_fields = ( set(field.name for field in imported._meta.fields) - set(field.name for field in TrackedModel._meta.fields) - {"trackedmodel_ptr"} ) for field in checked_fields: imported_value = getattr(imported, field) source_value = getattr(model, field) assert ( imported_value == source_value ), f"imported '{field}' ({imported_value} - {type(imported_value)}) does not match source '{field}' ({source_value} - {type(source_value)})" return imported
def test_replacement_importer_create(valid_user): target_regulation = factories.RegulationFactory.create() enacting_regulation = factories.RegulationFactory.create() measure_type_id = "AAAAAA" geographical_area_id = "GB" chapter_heading = "01" data = { "enacting_regulation": { "role_type": enacting_regulation.role_type, "regulation_id": enacting_regulation.regulation_id, }, "target_regulation": { "role_type": target_regulation.role_type, "regulation_id": target_regulation.regulation_id, }, "taric_template": "taric/replacement.xml", "update_type": UpdateType.CREATE.value, "measure_type_id": measure_type_id, "geographical_area_id": geographical_area_id, "chapter_heading": chapter_heading, } xml = generate_test_import_xml(data) process_taric_xml_stream(xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED.value) replacement = models.Replacement.objects.get( enacting_regulation__regulation_id=enacting_regulation.regulation_id, target_regulation__regulation_id=target_regulation.regulation_id, ) assert replacement.measure_type_id == measure_type_id assert replacement.geographical_area_id == geographical_area_id assert replacement.chapter_heading == chapter_heading assert replacement.enacting_regulation == enacting_regulation assert replacement.target_regulation == target_regulation
def test_goods_nomenclature_origin_importer_create(valid_user, date_ranges): origin = factories.GoodsNomenclatureFactory.create( update_type=UpdateType.CREATE.value, valid_between=date_ranges.big, origin__derived_from_goods_nomenclature__valid_between=date_ranges.adjacent_earlier_big, ) good = factories.GoodsNomenclatureFactory.create( update_type=UpdateType.CREATE.value, origin=None, ) origin_link = factories.GoodsNomenclatureOriginFactory.build( derived_from_goods_nomenclature=origin, new_goods_nomenclature=good, update_type=UpdateType.CREATE.value, ) xml = generate_test_import_xml( serializers.GoodsNomenclatureOriginSerializer( origin_link, context={"format": "xml"}, ).data, ) process_taric_xml_stream( xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED.value, ) db_link = models.GoodsNomenclatureOrigin.objects.get( new_goods_nomenclature__sid=good.sid, ) assert db_link db_good = models.GoodsNomenclature.objects.get(sid=good.sid) assert db_good.origins.get() == origin
def create_and_test_m2m_regulation( role_type: int, template: str, through_model: Union[Type[models.TrackedModel], Type[factories.TrackedModelMixin]], valid_user, update_type=UpdateType.CREATE, factory_kwargs=None, **extra_data): base_regulation = factories.RegulationFactory.create() if update_type != UpdateType.CREATE: test_regulation = factories.RegulationFactory.create( role_type=role_type) through_model.create(enacting_regulation=test_regulation, target_regulation=base_regulation, **(factory_kwargs or {})) through_model = through_model._meta.model else: test_regulation = factories.RegulationFactory.build( role_type=role_type) data = { "enacting_regulation": { "role_type": test_regulation.role_type, "regulation_id": test_regulation.regulation_id, "published_date": test_regulation.published_at, "official_journal_number": test_regulation.official_journal_number, "official_journal_page": test_regulation.official_journal_page, "effective_end_date": test_regulation.effective_end_date, "replacement_indicator": test_regulation.replacement_indicator, "community_code": test_regulation.community_code, "stopped": test_regulation.stopped, "information_text": "|".join([ test_regulation.information_text, test_regulation.public_identifier, test_regulation.url, ]), "approved": test_regulation.approved, }, "target_regulation": { "role_type": base_regulation.role_type, "regulation_id": base_regulation.regulation_id, }, "taric_template": template, "update_type": update_type, **extra_data, } xml = generate_test_import_xml(data) process_taric_xml_stream(xml, username=valid_user.username, status=WorkflowStatus.PUBLISHED.value) through_table_instance = through_model.objects.get( enacting_regulation__regulation_id=test_regulation.regulation_id, target_regulation__regulation_id=base_regulation.regulation_id, update_type=update_type, ) enacting_regulation = through_table_instance.enacting_regulation assert through_table_instance.target_regulation == base_regulation assert enacting_regulation.role_type == test_regulation.role_type assert enacting_regulation.regulation_id == test_regulation.regulation_id assert (enacting_regulation.official_journal_number == test_regulation.official_journal_number) assert (enacting_regulation.official_journal_page == test_regulation.official_journal_page) assert enacting_regulation.published_at == test_regulation.published_at assert enacting_regulation.information_text == test_regulation.information_text assert enacting_regulation.public_identifier == test_regulation.public_identifier assert enacting_regulation.url == test_regulation.url assert enacting_regulation.approved == test_regulation.approved assert (enacting_regulation.replacement_indicator == test_regulation.replacement_indicator) assert enacting_regulation.effective_end_date == test_regulation.effective_end_date assert enacting_regulation.stopped == test_regulation.stopped assert enacting_regulation.community_code == test_regulation.community_code assert enacting_regulation.stopped == test_regulation.stopped return through_table_instance