def parse_port_type(
wsdl: "Definition", xmlelement: etree._Element
) -> definitions.PortType:
"""Create a PortType object from a xml element.
Definition::
<wsdl:definitions .... >
<wsdl:portType name="nmtoken">
<wsdl:operation name="nmtoken" .... /> *
</wsdl:portType>
</wsdl:definitions>
:param wsdl: The parent definition instance
:param xmlelement: The XML node
"""
name = qname_attr(xmlelement, "name", wsdl.target_namespace)
assert name is not None
operations = {} # type: typing.Dict[str, definitions.AbstractOperation]
for elm in xmlelement.findall("wsdl:operation", namespaces=NSMAP):
operation = parse_abstract_operation(wsdl, elm)
if operation:
operations[operation.name] = operation
return definitions.PortType(name, operations)
def move_field_from_element(
original_parent: etree._Element,
to_remove: str,
replacement_parent: etree._Element = None,
) -> None:
""" An XML utility function to move or remove a field from an element.
If replacement_parent is None, this performs a removal.
Otherwise, it performs a replacement.
Args:
original_parent (etree._Element): The initial parent of the field to be moved/removed.
to_remove (str): The name of the field to be moved/removed. None means to move the root.
None can only be used to move and not remove.
replacement_parent (etree._Element, optional): The new parent for the field to be moved.
Defaults to None.
"""
if to_remove is not None:
elements_to_move = original_parent.findall(to_remove)
if elements_to_move is not None and len(elements_to_move) > 0:
for elem in elements_to_move:
elem.getparent().remove(elem)
if replacement_parent is not None:
# Replace the element
for elem in elements_to_move:
replacement_parent.append(elem)
else:
# We want to move the original parent here.
element_to_move = original_parent
if replacement_parent is not None or to_remove is None:
# Replace the element
replacement_parent.append(element_to_move)
def primitive_element_to_dto(
primitive_element: _Element,
rule_eval: Optional[rule.RuleInEffectEval] = None,
) -> CibResourcePrimitiveDto:
if rule_eval is None:
rule_eval = rule.RuleInEffectEvalDummy()
return CibResourcePrimitiveDto(
id=str(primitive_element.attrib["id"]),
agent_name=ResourceAgentNameDto(
standard=str(primitive_element.attrib["class"]),
provider=primitive_element.get("provider"),
type=str(primitive_element.attrib["type"]),
),
description=primitive_element.get("description"),
operations=[
op_element_to_dto(op_element, rule_eval)
for op_element in primitive_element.findall("operations/op")
],
meta_attributes=[
nvpair_multi.nvset_element_to_dto(nvset, rule_eval)
for nvset in nvpair_multi.find_nvsets(primitive_element,
nvpair_multi.NVSET_META)
],
instance_attributes=[
nvpair_multi.nvset_element_to_dto(nvset, rule_eval)
for nvset in nvpair_multi.find_nvsets(primitive_element,
nvpair_multi.NVSET_INSTANCE)
],
utilization=[
nvpair_multi.nvset_element_to_dto(nvset, rule_eval)
for nvset in nvpair_multi.find_nvsets(
primitive_element, nvpair_multi.NVSET_UTILIZATION)
],
)
def _get_ip(cls, xhost: etree._Element) -> (str, str):
"""Get and return ip address from a nmap result xml 'host' section.
return: (ip, iptype)
iptype: ipv4/ipv6"""
ipaddr = None
iptype = None
if xhost is None:
return
xaddrs = xhost.findall("address")
if xaddrs is None or len(xaddrs) < 1:
return
for xaddr in xaddrs:
# get ip address type from the ipaddr type attribute
xiptp = xaddr.get("addrtype")
if not isinstance(
xiptp, str) or xiptp == '' or not xiptp.startswith('ip'):
continue
iptype = xiptp
# get ip address from the ipaddr attribute
xip = xaddr.get("addr")
if not isinstance(xip, str) or xip is None or xip == '':
return
ipaddr = xip
return (ipaddr, iptype)
def process_aux(self, conf_tree: etree._Element,
rule_tree: etree._Element):
for conf_elem in conf_tree.iterchildren(tag=etree.Element):
conf_tag = conf_elem.tag
rule_elem = rule_tree.findall(conf_tag)
if len(rule_elem
) > 0: # If the entry corresponds to one or more rules
if len(rule_elem) == 1:
rule_elem = rule_elem[0]
action_attr = rule_elem.get('action', default=None)
if action_attr is not None and action_attr == 'no-translate': # If labeled "no translate", skip
pass
elif len(conf_elem) > 0: # If the config file has subelements
if len(rule_elem) > 0: # If the rules also have subelements
self.process_aux(conf_elem,
rule_elem) # recursive call
else: # If no corresponding elements in rules
logging.warning(
f'Mismatch between element contents: {conf_elem} and rule: {rule_elem}'
)
else: # Otherwise, it is a key-value pair
self.add_property(rule_elem.text, conf_elem.text)
else: # If the rules have a list,
self.process_multi(
conf_elem, rule_elem
) # We are distinguishing cases based on attribute
else: # If the key isn't in the rules, ignore it but warn because it is probably an error.
logging.warning(f"Tag {conf_tag} is not in element rules.")
def get_sd_person_changed(
identifier_type: IdType, identifier: str, xml_root: _Element
) -> SdPersonChange:
# Get the request date timestamps
request_structure = xml_root.find("RequestStructure")
activation_date = request_structure.find("ActivationDate").text.strip()
deactivation_date = request_structure.find("DeactivationDate").text.strip()
def id_match(id_type: IdType, id_: str, person: _Element) -> bool:
if id_type == IdType.CPR:
id_element = person.find(id_type.value)
else:
employment_element = person.find("Employment")
id_element = employment_element.find(id_type.value)
if id_ == id_element.text.strip():
return True
return False
person_elements = xml_root.findall("Person")
persons = filter(partial(id_match, identifier_type, identifier), person_elements)
sd_person_changed = SdPersonChange(
start_date=activation_date, end_date=deactivation_date, change=only(persons)
)
return sd_person_changed
def get_list_of_tag_elements(tags_section: _Element) -> List[_Element]:
"""
Get list of tag elements from cib.
tags_section -- element tags
"""
return tags_section.findall(TAG_TAG)
def delete_arc(app: et._Element, arc_from: str, arc_to: str):
arcs = app.findall('note/graph/arc')
for arc in arcs:
if arc.attrib['from'] == arc_from and arc.attrib['to'] == arc_to:
arc.getparent().remove(arc)
all_arcs, _ = get_arcs(app)
return all_arcs, app
def parse_messages(self, doc: etree._Element):
"""
Definition::
<definitions .... >
<message name="nmtoken"> *
<part name="nmtoken" element="qname"? type="qname"?/> *
</message>
</definitions>
:param doc: The source document
:type doc: lxml.etree._Element
"""
result = {}
for msg_node in doc.findall("wsdl:message", namespaces=NSMAP):
try:
msg = parse.parse_abstract_message(self, msg_node)
except IncompleteMessage as exc:
warnings.warn(str(exc))
else:
result[msg.name.text] = msg
logger.debug("Adding message: %s", msg.name.text)
return result
def _get_ports(cls, xhost: etree._Element, task: IscanTask,
level: int) -> iter:
"""parse xml, yield return PortInfo"""
# get ip address
ip, iptype = cls._get_ip(xhost)
if (not isinstance(ip, str) or ip == "" or not isinstance(iptype, str)
or iptype == ""):
return
# hostnames
hostnames: list = cls._get_hostnames(xhost)
# os
os, oscpe = cls._get_os(xhost)
# uptime
xuptime = xhost.get("uptime")
uptime: int = None
if not xuptime is None:
seconds = xuptime.get("seconds")
if not seconds is None:
try:
uptime = int(seconds)
except Exception:
pass
# open ports
xports = xhost.findall("ports/port")
if xports is None or len(xports) < 1:
return
for xp in xports:
try:
if not cls._check_port_open(xp):
continue
transprotocol = xp.get("protocol")
port: int = int(xp.get("portid").strip())
portinfo: PortInfo = PortInfo(task, level, ip, port,
transprotocol)
portinfo.set_ips(ip)
if not os is None and not os == "":
portinfo.os = os
if not oscpe is None and len(oscpe) > 0:
portinfo.set_cpe(*oscpe)
if not uptime is None:
portinfo.uptime = uptime
portinfo.set_domain(*hostnames)
xservice = xp.find("service")
if not xservice is None:
cls._get_port_service(xp, portinfo)
yield portinfo
except Exception:
cls._logger.debug("Get ports warn: {}".format(
traceback.format_exc()))
def delete_rdg(app: et._Element, rdg_n):
rdgs = app.findall('rdg')
for rdg in rdgs:
if rdg.attrib['n'] == rdg_n:
rdg.attrib.pop('type')
break
all_rdgs = get_all_rdgs(app)
return app, all_rdgs
def bl_find_heroes(parsed: etree._Element):
# Start the dict.
built_dict = {"role": "", "difficulty": "", "abilities": {}}
difficulty = len(parsed.findall(".//span[@class='star']"))
role = parsed.xpath(".//h4[@class='h2 hero-detail-role-name']")[0].text
_abilities = parsed.findall(".//div[@class='hero-ability-descriptor']")
abilities = {}
for ability in _abilities:
name, description = ability[0].text, ability[1].text
abilities[name] = description
built_dict["difficulty"] = difficulty
built_dict["role"] = role
built_dict["abilities"] = abilities
return built_dict
def bl_find_heroes(parsed: etree._Element):
# Start the dict.
built_dict = {"role": "", "difficulty": "", "abilities": {}}
difficulty = len(parsed.findall(".//span[@class='star']"))
role = parsed.xpath(".//h4[@class='h2 hero-detail-role-name']")[0].text
_abilities = parsed.findall(".//div[@class='hero-ability-descriptor']")
abilities = {}
for ability in _abilities:
name, description = ability[0].text, ability[1].text
abilities[name] = description
built_dict["difficulty"] = difficulty
built_dict["role"] = role
built_dict["abilities"] = abilities
return built_dict
def export_with_set(element: _Element) -> Dict[str, Any]:
return {
"resource_sets": [
resource_set.export(resource_set_item)
for resource_set_item in element.findall(".//resource_set")
],
"options":
export_attributes(element),
}
def parse_tree(tree: etree._Element,
annotation: Annotation) -> Dict[str, Source]:
sources = {}
for source in tree.findall('source'):
src = Source.parse(source,
subcls=Sentence,
annotation=annotation,
ignored_tags={'title'})
sources[src.id] = src
return sources
def parse_abstract_message(
wsdl, xmlelement: etree._Element) -> definitions.AbstractMessage:
"""Create an AbstractMessage object from a xml element.
Definition::
<definitions .... >
<message name="nmtoken"> *
<part name="nmtoken" element="qname"? type="qname"?/> *
</message>
</definitions>
:param wsdl: The parent definition instance
:type wsdl: zeep.wsdl.wsdl.Definition
:param xmlelement: The XML node
:type xmlelement: lxml.etree._Element
:rtype: zeep.wsdl.definitions.AbstractMessage
"""
tns = wsdl.target_namespace
message_name = qname_attr(xmlelement, "name", tns)
if not message_name:
raise IncompleteMessage(
"Message element is missing required name attribute")
parts = []
for part in xmlelement.findall("wsdl:part", namespaces=NSMAP):
part_name = part.get("name")
part_element = qname_attr(part, "element")
part_type = qname_attr(part, "type")
try:
if part_element is not None:
part_element = wsdl.types.get_element(part_element)
if part_type is not None:
part_type = wsdl.types.get_type(part_type)
except (NamespaceError, LookupError):
raise IncompleteMessage(
("The wsdl:message for %r contains an invalid part (%r): "
"invalid xsd type or elements") %
(message_name.text, part_name))
part = definitions.MessagePart(part_element, part_type)
parts.append((part_name, part))
# Create the object, add the parts and return it
msg = definitions.AbstractMessage(message_name)
for part_name, part in parts:
msg.add_part(part_name, part)
return msg
def check_tags(data: etree._Element) -> bool:
names = set([
'date', 'docnum', 'doctype', 'subject', 'body', 'image', 'sign',
'signtitle', 'missingtext'
])
elements = data.findall('.//')
tags = set([i.tag for i in elements])
diff = tags.difference(names)
if len(diff) != 0:
raise AttributeError(
str(diff) + " are mislabeled / do not follow protocols.")
return True
def from_lxml_element(cls,
element: _Element) -> "VehicleMonitoringDelivery":
activites = [
VehicleActivity.from_lxml_element(element)
for element in element.findall("x:VehicleActivity",
namespaces=_NSMAP)
]
return cls(
request_message_ref=element.findtext("x:RequestMessageRef",
namespaces=_NSMAP),
response_timestamp=element.findtext("x:ResponseTimestamp",
namespaces=_NSMAP),
shortest_possible_cycle=element.findtext("x:ShortestPossibleCycle",
namespaces=_NSMAP),
valid_until=element.findtext("x:ValidUntil", namespaces=_NSMAP),
vehicle_activities=activites,
)
def parse(self, root: etree._Element) -> AnnotationCollection:
"""
Parse XML tree and return a collection of annotations.
:param root: The XML tree root.
:type root: etree._Element
:return: A collection of parsed annotations.
:rtype: AnnotationCollection
"""
annotations = []
for raw_viewstate in root.findall("ndpviewstate"):
viewstate = self._parse_viewstate(raw_viewstate)
annotations.append(viewstate)
collection = AnnotationCollection(annotations)
return collection
def _get_host(cls, xhost: etree._Element, task: IscoutTask,
level: int) -> RangeCHost:
"""parse xml, yield return RangeCHost"""
host: RangeCHost = None
try:
# get ip address
ip, iptype = cls._get_ip(xhost)
if not isinstance(ip, str) or ip == '':
return host
host: RangeCHost = RangeCHost(task, level, ip)
host.iptype = iptype
# open ports
xports = xhost.findall("ports/port")
if xports is None or len(xports) < 1:
return host
for xp in xports:
try:
if not cls._check_port_open(xp):
continue
port: int = int(xp.get("portid").strip())
transprotocol = xp.get("protocol")
service: str = None
xservice = xp.find('service')
if not xservice is None:
service = xservice.get("name")
host.set_port(port, transprotocol, service)
except Exception:
cls._logger.debug("Get ports warn: {}".format(
traceback.format_exc()))
except Exception:
cls._logger.error(
"Parse one alive host error: taskid={} batchid={}\nerror:{}".
format(task.taskid, task.batchid, traceback.format_exc()))
return host
def tag_element_to_dict(
tag_element: _Element,
) -> Dict[str, Union[str, Iterable[str]]]:
"""
Convert tag element to the dict structure
tag_element -- single tag element that contains obj_ref elements
{
"tag_id": "tag1",
"idref_list": ["i1", "i2"],
},
"""
return {
# NOTE: .get("id", default="") for typing there always be an id
"tag_id": str(tag_element.get("id", default="")),
"idref_list": [
str(obj_ref.get("id", default=""))
for obj_ref in tag_element.findall(TAG_OBJREF)
],
}
def get_all_apps(ab: et._Element,
ignore: dict) -> Union[List[int], List[Tuple[int]]]:
'''returns a list of ints and/or tuples of ints'''
apps = ab.findall('app')
if apps is None:
return None
app_units = []
for app in apps:
if ignore['lac'] is True:
rdgs = app.findall('rdg')
if len(rdgs) == 2 and rdgs[1].get('type') == 'lac':
continue
if ignore['subr'] is True:
rdgs = app.findall('rdg')
if len(rdgs) == 2 and rdgs[1].get('type') == 'subreading':
continue
if app.get('from') == app.get('to'):
app_units.append(int(app.get('from')))
else:
app_units.append((int(app.get('from')), int(app.get('to'))))
return app_units
def get_all_rdgs(app: et._Element) -> List[Dict[str, str]]:
'''returns a list of dicts; no _Elements'''
if app is None:
return None
readings = []
rdgs = app.findall('rdg')
for rdg in rdgs:
wits = rdg.get('wit')
# wits = sorted(wits.split()) # this was an attempt to sort mss maj>min, but it didn't work.
# wits = ' '.join(wits)
if rdg.get('type') == 'subreading':
rdg_type = 'subr'
else:
rdg_type = rdg.get('type')
readings.append({
'name': rdg.get('n'),
'type': rdg_type,
'text': rdg.text,
'wits': wits
})
return readings
def parse_ports(self, doc: etree._Element) -> typing.Dict[str, PortType]:
"""Return dict with `PortType` instances as values
Definition::
<wsdl:definitions .... >
<wsdl:portType name="nmtoken">
<wsdl:operation name="nmtoken" .... /> *
</wsdl:portType>
</wsdl:definitions>
:param doc: The source document
:type doc: lxml.etree._Element
"""
result = {}
for port_node in doc.findall("wsdl:portType", namespaces=NSMAP):
port_type = parse.parse_port_type(self, port_node)
result[port_type.name.text] = port_type
logger.debug("Adding port: %s", port_type.name.text)
return result
def find_tag(data: etree._Element, tag: str, all=False) -> dict:
"""
Handles missing attribute errors with default inputs
"""
d = {}
tag_search = './/' + tag # to enable recursive search
if all is False:
try:
tag_node = data.find(tag_search)
d['text'] = get_text(tag_node)
d['line_num'] = tag_node.sourceline
except AttributeError:
d['text'] = None
d['line_num'] = None
return d
else:
for i, j in enumerate(data.findall(tag_search)):
d[i] = {'text': j.text, 'line_num': j.sourceline}
return d
def __init__(self, root: ET._Element):
self.patterns = {}
self.patterns_first = {}
# Identify block patterns.
# this is a block feature extractor..
for page in root.findall(f".//{ALTO}Page"):
blocks = page.findall(f".//{ALTO}TextBlock")
for block in blocks[:2] + blocks[-1:]:
text = misc.get_text(block)
first_line = text.split("\n")[0]
pattern = misc.get_pattern(first_line)
if len(pattern) <= 8:
continue
if pattern in self.patterns:
self.patterns[pattern] += 1
else:
self.patterns[pattern] = 1
self.patterns_first[pattern] = page.get("PHYSICAL_IMG_NR")
def parse_service(
wsdl: "Definition", xmlelement: etree._Element
) -> definitions.Service:
"""
Definition::
<wsdl:service name="nmtoken"> *
<wsdl:documentation .... />?
<wsdl:port name="nmtoken" binding="qname"> *
<wsdl:documentation .... /> ?
<-- extensibility element -->
</wsdl:port>
<-- extensibility element -->
</wsdl:service>
Example::
<service name="StockQuoteService">
<documentation>My first service</documentation>
<port name="StockQuotePort" binding="tns:StockQuoteBinding">
<soap:address location="http://example.com/stockquote"/>
</port>
</service>
:param wsdl: The parent definition instance
:param xmlelement: The XML node
"""
name = xmlelement.get("name")
ports = [] # type: typing.List[definitions.Port]
for port_node in xmlelement.findall("wsdl:port", namespaces=NSMAP):
port = parse_port(wsdl, port_node)
if port:
ports.append(port)
obj = definitions.Service(name)
for port in ports:
obj.add_port(port)
return obj
def _get_ordering_set_constraint_relation(
ord_set_const_el: _Element,
) -> RelationEntityDto:
attrs = cast(Mapping[str, str], ord_set_const_el.attrib)
members: Set[str] = set()
metadata: MutableMapping[str, Any] = dict(attrs)
metadata["sets"] = []
for rsc_set_el in ord_set_const_el.findall("resource_set"):
rsc_set: MutableMapping[str, Any] = dict(
id=str(rsc_set_el.get("id")),
metadata=dict(cast(Mapping[str, str], rsc_set_el.attrib)),
members=[],
)
metadata["sets"].append(rsc_set)
for rsc_ref in rsc_set_el.findall("resource_ref"):
rsc_id = str(rsc_ref.attrib["id"])
members.add(rsc_id)
rsc_set["members"].append(rsc_id)
return RelationEntityDto(
attrs["id"], ResourceRelationType.ORDER_SET, sorted(members), metadata
)
def _get_os(cls, xhost: etree._Element) -> (str, list):
"""get os type/name/version/cpe\n
return (osname, list(cpes))"""
xosmatches = xhost.findall('os/osmatch')
if xosmatches is None or len(xosmatches) < 1:
return (None, None)
cpes: list = []
xosmatch = None
curracuracy = 0
for xm in xosmatches:
# sort by accuracy
try:
# get the most accurate osname
accuracy = int(xm.get('accuracy'))
if accuracy > curracuracy:
curracuracy = accuracy
xosmatch = xm
# get all cpes
xcpes = xm.findall('osclass/cpe')
if xcpes is None or len(xcpes) < 1:
continue
for xcpe in xcpes:
if not xcpe.text in cpes:
cpes.append(xcpe.text)
except Exception:
pass
if xosmatch is None:
return (None, None)
name: str = xosmatch.get('name')
if not isinstance(name, str) or name == "":
return (None, None)
return (name, cpes)
def _get_hostnames(cls, xhost: etree._Element) -> list:
"""返回当前主机的hostnames列表"""
res: list = []
try:
if xhost is None:
return res
xhostnames = xhost.findall("hostnames/hostname")
if xhostnames is None or len(xhostnames) < 1:
return res
for xhostname in xhostnames:
hostname = xhostname.get("name")
if not isinstance(hostname, str) or hostname == "":
continue
if not hostname in res:
res.append(hostname)
except Exception:
cls._logger.debug("Get hostnames error:{}".format(
traceback.format_exc()))
return res
def check_is_without_duplication(
report_processor: ReportProcessor,
constraint_section: _Element,
element: _Element,
are_duplicate: Callable[[_Element, _Element], bool],
export_element: Callable[[_Element], Dict[str, Any]],
duplication_alowed: bool = False,
) -> None:
duplicate_element_list = [
duplicate_element
for duplicate_element in constraint_section.findall(".//" +
element.tag)
if (element is not duplicate_element
and are_duplicate(element, duplicate_element))
]
if not duplicate_element_list:
return
if report_processor.report_list([
ReportItem.info(
reports.messages.DuplicateConstraintsList(
element.tag,
[
export_element(duplicate_element)
for duplicate_element in duplicate_element_list
],
)),
ReportItem(
severity=reports.item.get_severity(
reports.codes.FORCE,
duplication_alowed,
),
message=reports.messages.DuplicateConstraintsExist([
str(duplicate.attrib["id"])
for duplicate in duplicate_element_list
]),
),
]).has_errors:
raise LibraryError()
def bl_get_all_heroes(parsed: etree._Element):
_heroes = parsed.findall(".//a[@class='hero-portrait-detailed']")
heroes = [hero.get("data-hero-id") for hero in _heroes]
return heroes
