def make_and_get_indent(indent, valid_user, depth):
data = {
"indented_goods_nomenclature": {
"sid": indent.indented_goods_nomenclature.sid,
"item_id": indent.indented_goods_nomenclature.item_id,
"suffix": indent.indented_goods_nomenclature.suffix,
},
"taric_template": "taric/goods_nomenclature_indent.xml",
"update_type": indent.update_type,
"sid": indent.sid,
"start_date": f"{indent.valid_between.lower:%Y-%m-%d}",
"indent": depth,
}
xml = generate_test_import_xml(data)
process_taric_xml_stream(
xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED.value,
)
return models.GoodsNomenclatureIndent.objects.filter(
sid=indent.sid,
).last()
def test_goods_nomenclature_successor_importer_create(valid_user, date_ranges):
good = factories.GoodsNomenclatureFactory(
update_type=UpdateType.CREATE.value,
valid_between=date_ranges.normal,
)
successor = factories.GoodsNomenclatureFactory(
update_type=UpdateType.CREATE.value,
valid_between=date_ranges.adjacent_later,
)
successor_link = factories.GoodsNomenclatureSuccessorFactory.build(
replaced_goods_nomenclature=good,
absorbed_into_goods_nomenclature=successor,
update_type=UpdateType.CREATE.value,
)
xml = generate_test_import_xml(
serializers.GoodsNomenclatureSuccessorSerializer(
successor_link,
context={"format": "xml"},
).data,
)
process_taric_xml_stream(
xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED.value,
)
db_link = models.GoodsNomenclatureSuccessor.objects.get(
replaced_goods_nomenclature__sid=good.sid,
)
assert db_link
db_good = models.GoodsNomenclature.objects.get(sid=good.sid)
assert db_good.successors.get() == successor
def run_import(xml, workflow_status=WorkflowStatus.PUBLISHED, record_group=None):
process_taric_xml_stream(
xml,
workbasket_id=None,
workbasket_status=workflow_status,
partition_scheme=get_partition_scheme(),
username=valid_user.username,
record_group=record_group,
)
def import_chunk(
chunk_pk: int,
workbasket_id: str,
workbasket_status: str,
partition_scheme_setting: str,
username: str,
record_group: Sequence[str] = None,
):
"""
Task for importing an XML chunk into the database.
This task must ensure the chunks workbasket_status reflects the import
process, whether it is currently running, errored or done. Once complete it
is also responsible for finding and setting up the next chunk tasks.
"""
partition_scheme = get_partition_scheme(partition_scheme_setting)
chunk = models.ImporterXMLChunk.objects.get(pk=chunk_pk)
logger.info(
"RUNNING CHUNK Batch: %s Record code: %s Chapter heading: %s Chunk number: %d",
chunk.batch.name,
chunk.record_code,
chunk.chapter,
chunk.chunk_number,
)
chunk.status = models.ImporterChunkStatus.RUNNING
chunk.save()
try:
process_taric_xml_stream(
BytesIO(chunk.chunk_text.encode()),
workbasket_id,
workbasket_status,
partition_scheme,
username,
record_group=record_group,
)
except Exception as e:
chunk.status = models.ImporterChunkStatus.ERRORED
chunk.save()
raise e
chunk.status = models.ImporterChunkStatus.DONE
chunk.save()
find_and_run_next_batch_chunks(
chunk.batch,
workbasket_id,
workbasket_status,
partition_scheme_setting,
username,
)
def test_goods_nomenclature_successor_importer_delete(valid_user, date_ranges):
good = factories.GoodsNomenclatureFactory(
update_type=UpdateType.CREATE.value,
valid_between=date_ranges.normal,
)
successor = factories.GoodsNomenclatureFactory(
update_type=UpdateType.CREATE.value,
valid_between=date_ranges.adjacent_later,
)
factories.GoodsNomenclatureSuccessorFactory(
replaced_goods_nomenclature=good,
absorbed_into_goods_nomenclature=successor,
update_type=UpdateType.CREATE.value,
)
updated_good = factories.GoodsNomenclatureFactory(
sid=good.sid,
item_id=good.item_id,
suffix=good.suffix,
version_group=good.version_group,
update_type=UpdateType.UPDATE.value,
valid_between=date_ranges.no_end,
)
successor_link = factories.GoodsNomenclatureSuccessorFactory.build(
replaced_goods_nomenclature=updated_good,
absorbed_into_goods_nomenclature=successor,
update_type=UpdateType.DELETE.value,
)
xml = generate_test_import_xml(
serializers.GoodsNomenclatureSuccessorSerializer(
successor_link,
context={"format": "xml"},
).data,
)
process_taric_xml_stream(
xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED.value,
)
db_link = models.GoodsNomenclatureSuccessor.objects.filter(
replaced_goods_nomenclature__sid=good.sid,
)
assert not db_link.latest_approved().exists()
assert db_link.latest_deleted().exists()
def test_replacement_importer_update(valid_user):
replacement = factories.ReplacementFactory.create()
data = {
"enacting_regulation": {
"role_type": replacement.enacting_regulation.role_type,
"regulation_id": replacement.enacting_regulation.regulation_id,
},
"target_regulation": {
"role_type": replacement.target_regulation.role_type,
"regulation_id": replacement.target_regulation.regulation_id,
},
"taric_template": "taric/replacement.xml",
"update_type": UpdateType.UPDATE.value,
"measure_type_id": replacement.measure_type_id,
"geographical_area_id": "UK",
"chapter_heading": replacement.chapter_heading,
}
xml = generate_test_import_xml(data)
process_taric_xml_stream(xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED.value)
replacements = models.Replacement.objects.filter(
enacting_regulation__regulation_id=replacement.enacting_regulation.
regulation_id,
target_regulation__regulation_id=replacement.target_regulation.
regulation_id,
)
assert replacements.count() == 2
replacement = replacements.get(update_type=UpdateType.UPDATE)
assert replacement.measure_type_id == replacement.measure_type_id
assert replacement.geographical_area_id == replacement.geographical_area_id
assert replacement.chapter_heading == replacement.chapter_heading
assert replacement.enacting_regulation == replacement.enacting_regulation
assert replacement.target_regulation == replacement.target_regulation
version_group = replacement.version_group
assert version_group.versions.count() == 2
assert (version_group == replacements.get(
update_type=UpdateType.CREATE).version_group)
assert version_group.current_version == replacement
def check(
model: Union[TrackedModel, Type[DjangoModelFactory]],
serializer: Type[TrackedModelSerializer],
) -> TrackedModel:
get_nursery().cache.clear()
settings.SKIP_WORKBASKET_VALIDATION = True
if isinstance(model, type) and issubclass(model, DjangoModelFactory):
model = model.build(update_type=UpdateType.CREATE)
assert isinstance(
model,
TrackedModel,
), "Either a factory or an object instance needs to be provided"
xml = generate_test_import_xml(
serializer(model, context={"format": "xml"}).data,
)
process_taric_xml_stream(
xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED,
)
db_kwargs = model.get_identifying_fields()
imported = model.__class__.objects.get_latest_version(**db_kwargs)
checked_fields = (
set(field.name for field in imported._meta.fields)
- set(field.name for field in TrackedModel._meta.fields)
- {"trackedmodel_ptr"}
)
for field in checked_fields:
imported_value = getattr(imported, field)
source_value = getattr(model, field)
assert (
imported_value == source_value
), f"imported '{field}' ({imported_value} - {type(imported_value)}) does not match source '{field}' ({source_value} - {type(source_value)})"
return imported
def test_replacement_importer_create(valid_user):
target_regulation = factories.RegulationFactory.create()
enacting_regulation = factories.RegulationFactory.create()
measure_type_id = "AAAAAA"
geographical_area_id = "GB"
chapter_heading = "01"
data = {
"enacting_regulation": {
"role_type": enacting_regulation.role_type,
"regulation_id": enacting_regulation.regulation_id,
},
"target_regulation": {
"role_type": target_regulation.role_type,
"regulation_id": target_regulation.regulation_id,
},
"taric_template": "taric/replacement.xml",
"update_type": UpdateType.CREATE.value,
"measure_type_id": measure_type_id,
"geographical_area_id": geographical_area_id,
"chapter_heading": chapter_heading,
}
xml = generate_test_import_xml(data)
process_taric_xml_stream(xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED.value)
replacement = models.Replacement.objects.get(
enacting_regulation__regulation_id=enacting_regulation.regulation_id,
target_regulation__regulation_id=target_regulation.regulation_id,
)
assert replacement.measure_type_id == measure_type_id
assert replacement.geographical_area_id == geographical_area_id
assert replacement.chapter_heading == chapter_heading
assert replacement.enacting_regulation == enacting_regulation
assert replacement.target_regulation == target_regulation
def test_goods_nomenclature_origin_importer_create(valid_user, date_ranges):
origin = factories.GoodsNomenclatureFactory.create(
update_type=UpdateType.CREATE.value,
valid_between=date_ranges.big,
origin__derived_from_goods_nomenclature__valid_between=date_ranges.adjacent_earlier_big,
)
good = factories.GoodsNomenclatureFactory.create(
update_type=UpdateType.CREATE.value,
origin=None,
)
origin_link = factories.GoodsNomenclatureOriginFactory.build(
derived_from_goods_nomenclature=origin,
new_goods_nomenclature=good,
update_type=UpdateType.CREATE.value,
)
xml = generate_test_import_xml(
serializers.GoodsNomenclatureOriginSerializer(
origin_link,
context={"format": "xml"},
).data,
)
process_taric_xml_stream(
xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED.value,
)
db_link = models.GoodsNomenclatureOrigin.objects.get(
new_goods_nomenclature__sid=good.sid,
)
assert db_link
db_good = models.GoodsNomenclature.objects.get(sid=good.sid)
assert db_good.origins.get() == origin
def create_and_test_m2m_regulation(
role_type: int,
template: str,
through_model: Union[Type[models.TrackedModel],
Type[factories.TrackedModelMixin]],
valid_user,
update_type=UpdateType.CREATE,
factory_kwargs=None,
**extra_data):
base_regulation = factories.RegulationFactory.create()
if update_type != UpdateType.CREATE:
test_regulation = factories.RegulationFactory.create(
role_type=role_type)
through_model.create(enacting_regulation=test_regulation,
target_regulation=base_regulation,
**(factory_kwargs or {}))
through_model = through_model._meta.model
else:
test_regulation = factories.RegulationFactory.build(
role_type=role_type)
data = {
"enacting_regulation": {
"role_type":
test_regulation.role_type,
"regulation_id":
test_regulation.regulation_id,
"published_date":
test_regulation.published_at,
"official_journal_number":
test_regulation.official_journal_number,
"official_journal_page":
test_regulation.official_journal_page,
"effective_end_date":
test_regulation.effective_end_date,
"replacement_indicator":
test_regulation.replacement_indicator,
"community_code":
test_regulation.community_code,
"stopped":
test_regulation.stopped,
"information_text":
"|".join([
test_regulation.information_text,
test_regulation.public_identifier,
test_regulation.url,
]),
"approved":
test_regulation.approved,
},
"target_regulation": {
"role_type": base_regulation.role_type,
"regulation_id": base_regulation.regulation_id,
},
"taric_template": template,
"update_type": update_type,
**extra_data,
}
xml = generate_test_import_xml(data)
process_taric_xml_stream(xml,
username=valid_user.username,
status=WorkflowStatus.PUBLISHED.value)
through_table_instance = through_model.objects.get(
enacting_regulation__regulation_id=test_regulation.regulation_id,
target_regulation__regulation_id=base_regulation.regulation_id,
update_type=update_type,
)
enacting_regulation = through_table_instance.enacting_regulation
assert through_table_instance.target_regulation == base_regulation
assert enacting_regulation.role_type == test_regulation.role_type
assert enacting_regulation.regulation_id == test_regulation.regulation_id
assert (enacting_regulation.official_journal_number ==
test_regulation.official_journal_number)
assert (enacting_regulation.official_journal_page ==
test_regulation.official_journal_page)
assert enacting_regulation.published_at == test_regulation.published_at
assert enacting_regulation.information_text == test_regulation.information_text
assert enacting_regulation.public_identifier == test_regulation.public_identifier
assert enacting_regulation.url == test_regulation.url
assert enacting_regulation.approved == test_regulation.approved
assert (enacting_regulation.replacement_indicator ==
test_regulation.replacement_indicator)
assert enacting_regulation.effective_end_date == test_regulation.effective_end_date
assert enacting_regulation.stopped == test_regulation.stopped
assert enacting_regulation.community_code == test_regulation.community_code
assert enacting_regulation.stopped == test_regulation.stopped
return through_table_instance
