def from_section(cls, alias, section: etree.ElementBase, header_size=1):
attributes = OrderedDict()
props: Iterable[etree.ElementBase] = section.find('Properties')
for prop in props:
attributes[prop.tag] = alias.get_data_type_from_section(
prop.find('Type'))
none_section = section.find('AllowNone')
allow_none = none_section is not None and none_section.text.strip() == 'true'
return cls(attributes, allow_none)
def from_section(cls, alias, section: etree.ElementBase, header_size):
child_type = alias.get_data_type_from_section(
section.find('of'))
none_section = section.find('AllowNone')
allow_none = none_section is not None and none_section.text.strip() == 'true'
size_section = section.find('size')
if size_section is not None:
array_size = int(size_section.text.strip())
else:
array_size = None
return cls(child_type, array_size=array_size, allow_none=allow_none, header_size=1)
def _get_header_size_from_section(self, section: etree.ElementBase):
"""
E.g.
<onEnterPreBattle>
<Arg>BLOB</Arg>
<VariableLengthHeaderSize>
<WarnLevel>none</WarnLevel>
</VariableLengthHeaderSize>
</onEnterPreBattle>
"""
if section.find('VariableLengthHeaderSize'):
return int(section.find('VariableLengthHeaderSize').text.strip())
return None
def from_section(cls, section: etree.ElementBase, alias):
"""
Create property object from xml section like following
<ownShipId>
<Type>ENTITY_ID</Type>
<Flags>ALL_CLIENTS</Flags>
<Default>0</Default>
</ownShipId>
"""
type_ = alias.get_data_type_from_section(section.find('Type'))
default = section.find('Default')
flags = section.find('Flags').text.strip()
return cls(section.tag, type_, flags=flags, default=default)
def construct_ziotype_data(session, log_scope,
document: etree.ElementBase) -> dict:
fields = document.find("velden")
omschrijving = trim_string(
session,
log_scope,
find(fields, "naam"),
80,
"omschrijving",
ObjectTypenKeys.resultaattypen,
)
return {
"informatieobjecttype_omschrijving":
omschrijving,
"volgnummer":
int(document.get("volgnummer")),
"richting":
get_choice_field(
session,
f"{log_scope} richting",
find(fields, "type", False),
RichtingChoices.values,
ObjectTypenKeys.zaakinformatieobjecttypen,
default=DEFAULT_RICHTING,
),
# TODO no mapping for non-required fields
# "statustype": "http://example.com"
}
def _extract_quote(self, postage_node: etree.ElementBase) -> QuoteDetails:
postage: RateRes.PostageType = RateRes.PostageType()
postage.build(postage_node)
currency = "USD"
services: List[RateRes.SpecialServiceType] = [
(lambda s: (s, s.build(svc)))(RateRes.SpecialServiceType())[0]
for svc in postage_node.xpath(".//*[local-name() = $name]",
name="SpecialService")
]
def get(key: str) -> Any:
return reduce(lambda r, v: v.text, postage_node.findall(key), None)
return QuoteDetails(
carrier=self.client.carrier_name,
service_name=None,
service_type=get("MailService"),
base_charge=None,
duties_and_taxes=None,
total_charge=float(postage_node.find("Rate").text),
currency=currency,
delivery_date=postage.CommitmentDate,
discount=None,
extra_charges=[
ChargeDetails(
name=SpecialService(str(svc.ServiceID)).name,
amount=svc.Price,
currency=currency,
) for svc in services
],
)
def find(el: etree.ElementBase, path: str, required=True) -> str:
"""find child element and return its text"""
result = el.find(path).text
if not result and required:
raise ParserException(f"the element with path {path} is empty")
else:
return result or ""
def from_section(cls, alias, section: etree.ElementBase, header_size):
type_section = section.find('Type')
if type_section is None:
child_type = Blob()
else:
child_type = alias.get_data_type_from_section(type_section)
return cls(child_type, header_size=1)
def from_section(cls, alias, section: etree.ElementBase, header_size):
type_section = section.find('Type')
if type_section is None:
logging.warning('%s does not provide type, what to do?', section)
return None
child_type = alias.get_data_type_from_section(type_section)
return cls(child_type, header_size=1)
def _find_with_ns(node: etree.ElementBase,
query: Text) -> List[etree.ElementBase]:
result = node.find(query, node.nsmap)
if result is None:
return []
if isinstance(result, etree.ElementBase):
return [result]
return list(result)
def construct_statustype_data(session, log_scope,
statustype: etree.ElementBase) -> dict:
fields = statustype.find("velden")
return {
"volgnummer": int(statustype.get("volgnummer")),
"omschrijving": find(fields, "naam"),
"omschrijvingGeneriek": find(fields, "naam-model", False),
"statustekst": find(fields, "bericht", False),
# "informeren": true
}
def __init__(self,
chapter_element: etree.ElementBase,
manga_title: str,
title_id: str,
group_id: Optional[int] = None):
chapter_link = chapter_element.find('div/a')
chapter_url = chapter_link.attrib['href']
chapter_title = chapter_link.text
if (m := self.chapter_number_regex.match(chapter_url)) is None:
raise ValueError('FoolSlide regex failed to find chapter numbers')
def from_section(cls, section: etree.ElementBase, alias):
args = []
if section.find('Args') is not None:
for item in section.find('Args'):
args.append(
MethodArgument(
name=item.tag,
type_=alias.get_data_type_from_section(item)))
else:
for item in section.findall('Arg'):
args.append(
MethodArgument(
type_=alias.get_data_type_from_section(item)))
header_section = section.find('VariableLengthHeaderSize')
try:
header_size = int(header_section.text.strip())
except (ValueError, AttributeError):
header_size = DEFAULT_HEADER_SIZE
return cls(section.tag, list(args), header_size)
def get_or_create_element_at(parent: etree.ElementBase, path: List[str]) -> etree.ElementBase:
if not path:
return parent
child = parent.find(TEI_NS_PREFIX + path[0])
if child is None:
LOGGER.debug('creating element: %s', path[0])
tag_expression = parse_tag_expression(path[0])
child = tag_expression.create_node(
element_maker=TEI_E
)
parent.append(child)
return get_or_create_element_at(child, path[1:])
def rss_document_to_feed(rss_url: str, tree: ElementBase) -> Feed:
# required rss channel items
title = tree.findtext("channel/title")
description = parse_description(tree.findtext("channel/description"))
link = tree.findtext("channel/link")
# optional rss channel items
category = tree.find("channel/category")
image_url = tree.find("channel/image/url")
image_title = tree.find("channel/image/title")
image_link = tree.find("channel/image/link")
return Feed(
url=rss_url.rstrip("/"),
title=title,
description=description,
link=link,
feed_source_type=FeedSourceType.RSS,
category=category.text if category is not None else None,
image_url=image_url.text if image_url is not None else None,
image_title=image_title.text if image_title is not None else None,
image_link=image_link.text if image_link is not None else None,
)
def replace_notelabel(el: ElementBase, text: str) -> None:
'修改脚注的标签'
while el.tag != 'a':
el = el.getparent()
if el.tag != 'a':
a = el.find('.//a')
if a is not None:
el = a
while len(el):
el = el[0]
el.text = text
def rdf_document_to_feed(rss_url: str, tree: ElementBase) -> Feed:
# required rss channel items
title = tree.findtext("{*}channel/{*}title")
description = parse_description(tree.findtext("{*}channel/{*}description"))
link = tree.findtext("{*}channel/{*}link")
# optional rss channel items
category = tree.find("{*}channel/{*}category")
image_tag = tree.find("{*}channel/{*}image")
image_url = (
image_tag.get("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource")
if image_tag is not None else None)
return Feed(
url=rss_url.rstrip("/"),
title=title,
description=description,
link=link,
feed_source_type=FeedSourceType.RDF,
category=category.text if category is not None else None,
image_url=image_url if image_url is not None else None,
image_title=None,
image_link=None,
)
def get_procestype(process: etree.ElementBase, processtype_year: int) -> str:
# use vernietigingsgrondslag from the first resultaattype of this zaaktype
resultaattype = process.find("resultaattypen/resultaattype")
resultaat_number = get_resultaat_number(resultaattype)
if not resultaat_number:
# TODO what to do here?
return ""
procestype_number = int(resultaat_number.split(".")[0])
procestype = [
p for p in get_procestypen(processtype_year)
if p["nummer"] == procestype_number
][0]
return procestype["url"]
def construct_roltype_data(session, log_scope,
roltype: etree.ElementBase) -> dict:
# We could also use /dsp/rolsoorten/*/rolsoort, it doesn't matter much for our
# case.
fields = roltype.find("velden")
return {
"omschrijving":
find(fields, "naam"),
"omschrijvingGeneriek":
get_choice_field(
session,
f"{log_scope} omschrijvingGeneriek",
find(fields, "naam-model", False),
RolOmschrijving.values,
ObjectTypenKeys.roltypen,
default=DEFAULT_ROL_OMSCHRIVING,
extra_mapping={
"zaakcoördinator": RolOmschrijving.zaakcoordinator,
},
),
}
def construct_iotype_data(session, log_scope,
document: etree.ElementBase) -> dict:
fields = document.find("velden")
omschrijving = find(fields, "naam")
# dirty trim for log_scope
log_scope = f"{log_scope} iotype '{omschrijving[:80].strip()}'"
# clean trim with reporting
omschrijving = trim_string(
session,
log_scope,
omschrijving,
80,
"omschrijving",
ObjectTypenKeys.resultaattypen,
)
iotype_data = {
"omschrijving":
omschrijving,
# FIXME this field is always empty in the example xml
"vertrouwelijkheidaanduiding":
get_choice_field(
session,
f"{log_scope} vertrouwelijkheidaanduiding",
find(fields, "vertrouwelijkheid", False),
VertrouwelijkheidsAanduidingen.values,
ObjectTypenKeys.informatieobjecttypen,
default=DEFAULT_VERTROUWELIJKHEID,
),
# begin data would be set during merging different iotypen later
"beginGeldigheid":
session.job.start_date.isoformat(),
"eindeGeldigheid":
None,
}
return iotype_data
def add(self, name: str, figure_template: et.ElementBase,
diagram_root: et.ElementBase):
"""
Добавляет фигуру на схему
"""
# скопируем xml-объект фигуры из шаблона
xml_object_template: et.ElementBase = figure_template
xml_object: et.ElementBase = deepcopy(xml_object_template)
cell: et.ElementBase = xml_object.getchildren()[0]
geometry: et.ElementBase = cell.getchildren()[0]
# создадим фигуру
child_figure: Figure = Figure(self, xml_object, name,
figure_template.attrib['type'])
# инкремент ключа в случае совпадения с уже имеющимся
if child_figure.id in g_figures.keys():
key_incrementer: int = 2
while f'{child_figure.id}-{str(key_incrementer)}' in g_figures.keys(
):
key_incrementer += 1
child_figure.id += '-' + str(key_incrementer)
g_figures[
child_figure.
id] = child_figure # добавим значение по уникальному ID в словарь фигур
if 'max_rows' in figure_template.attrib:
child_figure.max_rows = int(figure_template.attrib['max_rows'])
del xml_object.attrib['max_rows']
if 'padding' in figure_template.attrib:
child_figure.padding = int(figure_template.attrib['padding'])
del xml_object.attrib['padding']
# добавим фигуру в схему
xml_object.attrib['id'] = child_figure.id
exist_xml_object: et.ElementBase = diagram_root.find(
f'object[@id="{child_figure.id}"]')
if exist_xml_object is None:
diagram_root.append(xml_object)
else:
# фиксируем положение фигуры
child_figure.is_exists = True
exist_cell: et.ElementBase = exist_xml_object.getchildren()[0]
exist_geometry: et.ElementBase = exist_cell.getchildren()[0]
# if 'edge' in exist_cell.attrib:
# для соединений сохраним геометрию целиком
new_geometry = deepcopy(exist_geometry)
cell.replace(geometry, new_geometry)
geometry = new_geometry
# else:
# # для фигур фиксируем положение (x, y)
# geometry.attrib['x'] = exist_geometry.attrib['x']
# geometry.attrib['y'] = exist_geometry.attrib['y']
diagram_root.replace(exist_xml_object, xml_object)
child_figure.geometry = geometry
if 'edge' in cell.attrib:
# соединение
cell.attrib['source'] = self.xml_object.attrib[
'id'] # источник - родительская фигура
self.connectors.append(child_figure)
return child_figure
# далее обрабатываем фигуры, не являющиеся соединениями
# имя соединения в глобальный список не добавляем
g_figures[
name] = child_figure # наименование, может быть не уникально, перезаписывается
# пересчитаем оценку площади фигур
child_figure.min_area = float(geometry.attrib['width']) * float(
geometry.attrib['height'])
parent: Figure = self
while parent is not None:
parent.min_area += child_figure.min_area
parent = parent.parent
if 'align' in xml_object.attrib:
# фигура "прилеплена" к границе родителя
align_type: str = xml_object.attrib['align']
if self.aligned_figures.get(align_type) is None:
self.aligned_figures[align_type] = []
self.aligned_figures[align_type].append(xml_object)
else:
# фигура вложена в контейнер
self.inner_figures.append(child_figure)
return child_figure
def _parse_section(self, section: etree.ElementBase):
self._parse_implements(section.find("Implements"))
self._parse_properties(section.find("Properties"))
self._parse_volatile(section.find("Volatile"))
def get(parent_element: etree.ElementBase, tag: str) -> etree.ElementBase:
namespace = [ns for ns in parent_element.nsmap.values()]
return parent_element.find(tag) if not namespace else parent_element.find('ns:%s' % tag, {'ns': '%s' % namespace[0]})
def construct_zaaktype_data(session, log_scope, process: etree.ElementBase,
processtype_year: int) -> dict:
fields = process.find("velden")
omschrijving = trim_string(
session,
log_scope,
find(fields, "kernomschrijving"),
80,
"omschrijving",
ObjectTypenKeys.zaaktypen,
)
omschrijvingGeneriek = trim_string(
session,
log_scope,
find(fields, "model-kernomschrijving", False),
80,
"omschrijvingGeneriek",
ObjectTypenKeys.zaaktypen,
)
indicatie_intern_of_extern = ("extern" if "extern" in find(
fields, "zaaktype-categorie", False).lower() else "intern")
handeling_initiator = value_or_default(
session,
f"{log_scope} handelingInitiator",
find(fields, "zaaktype-naam/structuur/handeling-initiator", False),
DEFAULT_HANDELING_INITIATOR,
ObjectTypenKeys.zaaktypen,
)
aanleiding = value_or_default(
session,
f"{log_scope} aanleiding",
find(fields, "aanleiding", False),
DEFAULT_AANLEIDING,
ObjectTypenKeys.zaaktypen,
)
onderwerp = value_or_default(
session,
f"{log_scope} onderwerp",
find(fields, "zaaktype-naam/structuur/onderwerp", False),
DEFAULT_ONDERWERP,
ObjectTypenKeys.zaaktypen,
)
handeling_behandelaar = value_or_default(
session,
f"{log_scope} handeling_behandelaar",
find(fields, "zaaktype-naam/structuur/handeling-behandelaar", False),
DEFAULT_HANDELING_BEHANDELAAR,
ObjectTypenKeys.zaaktypen,
)
servicenorm = get_duration(
find(fields, "afdoeningstermijn"),
find(fields, "afdoeningstermijn-eenheid"),
)
doorlooptijd = get_duration(
find(fields, "wettelijke-afdoeningstermijn", False),
find(fields, "wettelijke-afdoeningstermijn-eenheid", False),
)
if not doorlooptijd:
doorlooptijd = get_duration(
find(fields, "afdoeningstermijn"),
find(fields, "afdoeningstermijn-eenheid"),
)
session.log_info(
f'{log_scope} Used "afdoeningstermijn" ({doorlooptijd}) for "Zaaktype.doorlooptijd": Import has no value for "wettelijke-afdoeningstermijn".',
ObjectTypenKeys.zaaktypen,
)
verlengings_termijn = get_duration(
find(fields, "wettelijke-verdagingstermijn", False),
find(fields, "wettelijke-verdagingstermijn-eenheid", False),
)
return {
"identificatie":
process.get("id"),
"omschrijving":
omschrijving,
"omschrijvingGeneriek":
omschrijvingGeneriek,
"vertrouwelijkheidaanduiding":
get_choice_field(
session,
f"{log_scope} vertrouwelijkheidaanduiding",
find(fields, "vertrouwelijkheid", False),
VertrouwelijkheidsAanduidingen.values,
ObjectTypenKeys.zaaktypen,
default=DEFAULT_VERTROUWELIJKHEID,
required=True,
),
"doel":
find(fields, "naam"),
"aanleiding":
aanleiding,
"toelichting":
find(fields, "toelichting-proces", False),
"indicatieInternOfExtern":
indicatie_intern_of_extern,
"handelingInitiator":
handeling_initiator,
"onderwerp":
onderwerp,
"handelingBehandelaar":
handeling_behandelaar,
"doorlooptijd":
doorlooptijd,
"opschortingEnAanhoudingMogelijk":
get_boolean(find(fields, "aanhouden-mogelijk", False)),
"verlengingMogelijk":
bool(verlengings_termijn),
"verlengingstermijn":
verlengings_termijn,
"trefwoorden":
get_array(find(fields, "lokale-trefwoorden", False)), # always empty?
"publicatieIndicatie":
get_boolean(find(fields, "publicatie-indicatie", False)),
"publicatietekst":
find(fields, "publicatietekst", False),
"verantwoordingsrelatie":
get_array(find(fields, "verantwoordingsrelatie",
False)), # always empty?
"selectielijstProcestype":
get_procestype(process, processtype_year),
"referentieproces": {
"naam": find(fields, "ztc-procestype")
},
# Set during `load_data`
# "catalogus": "",
"beginGeldigheid":
session.job.start_date.isoformat(),
"eindeGeldigheid":
None,
"versiedatum":
get_date(find(fields, "actueel-van")),
"servicenorm":
servicenorm,
"productenOfDiensten": [],
"gerelateerdeZaaktypen": [],
"besluittypen": [],
# "deelzaaktypen": [],
}
def _parse_section(self, section: etree.ElementBase):
super(EntityDef, self)._parse_section(section)
self._parse_client_methods(section.find("ClientMethods"))
self._parse_cell_methods(section.find("CellMethods"))
self._parse_base_methods(section.find("BaseMethods"))
def construct_resultaattype_data(session, log_scope,
resultaattype: etree.ElementBase,
processtype: str) -> dict:
fields = resultaattype.find("velden")
toelichting = find(fields, "toelichting", False)
afleidingswijze = get_choice_field(
session,
f"{log_scope} afleidingswijze",
find(fields, "brondatum-archiefprocedure", False),
BrondatumArchiefprocedureAfleidingswijze.values,
ObjectTypenKeys.resultaattypen,
default=DEFAULT_AFLEIDINGSWIJZE,
)
if afleidingswijze == BrondatumArchiefprocedureAfleidingswijze.afgehandeld:
datumkenmerk = ""
elif ":" in toelichting:
datumkenmerk = toelichting.split(":")[0]
else:
datumkenmerk = toelichting.split(",")[-1].strip()
datumkenmerk = trim_string(
session,
log_scope,
datumkenmerk,
80,
"datumkenmerk",
ObjectTypenKeys.resultaattypen,
)
omschrijving = trim_string(
session,
log_scope,
find(fields, "naam"),
20,
"naam",
ObjectTypenKeys.resultaattypen,
)
resultaattype_data = {
"omschrijving":
omschrijving,
"resultaattypeomschrijving":
get_resultaattype_omschrijving(session, log_scope, resultaattype),
"selectielijstklasse":
get_resultaat(session, log_scope, resultaattype, processtype),
"toelichting":
toelichting,
"archiefnominatie":
get_choice_field(
session,
f"{log_scope} archiefnominatie",
find(fields, "waardering", False),
Archiefnominatie.values,
ObjectTypenKeys.resultaattypen,
default=DEFAULT_ARCHIEFNOMINATIE,
extra_mapping={
"bewaren": Archiefnominatie.blijvend_bewaren,
},
),
"archiefactietermijn":
get_duration(
find(fields, "bewaartermijn", False),
find(fields, "bewaartermijn-eenheid", False),
),
"brondatumArchiefprocedure": {
"afleidingswijze": afleidingswijze,
"datumkenmerk": datumkenmerk,
# FIXME fixed values are set to prevent 500 error
"einddatumBekend": False,
"objecttype": "",
"registratie": "",
"procestermijn": None,
},
}
brondatum_params = resultaattype_data["brondatumArchiefprocedure"]
if brondatum_params["afleidingswijze"] == "ander_datumkenmerk":
brondatum_params["objecttype"] = "overige"
brondatum_params["registratie"] = "TODO"
session.log_info(
f"{log_scope} resultaattype '{resultaattype_data['omschrijving']}' doesn't have "
f"brondatumArchiefprocedure.objecttype. It will be set as 'overige'",
ObjectTypenKeys.resultaattypen,
)
session.log_info(
f"{log_scope} resultaattype '{resultaattype_data['omschrijving']}' doesn't have "
f"brondatumArchiefprocedure.registratie. It will be set as 'TODO'",
ObjectTypenKeys.resultaattypen,
)
return resultaattype_data