def test_json_serializer_with_presence_adapter(self):
from FreeTAKServer.controllers.RestMessageControllers.SendPresenceController import SendPresenceController
from FreeTAKServer.controllers.serializers.api_adapters.api_adapters import PresenceAdapter
basejson = {
"how": "nonCoT",
"name": "testing",
"latitude": "34.5677889",
"longitude": "12.345678",
"role": "Team Member",
"team": "Cyan"
}
jsonobj = JsonController().serialize_presence_post(basejson)
simpleCoTObject = SendPresenceController(jsonobj).getCoTObject()
xml_legacy = etree.fromstring(simpleCoTObject.xmlString)
xml_legacy.attrib['uid'] = '123'
xml_legacy.attrib['time'] = '24'
xml_legacy.attrib['start'] = '849'
xml_legacy.attrib['stale'] = '3333'
newObject = PresenceAdapter().from_api_to_fts_object(basejson)
xml_updated = etree.fromstring(newObject.xmlString)
xml_updated.attrib['uid'] = '123'
xml_updated.attrib['time'] = '24'
xml_updated.attrib['start'] = '849'
xml_updated.attrib['stale'] = '3333'
xml_legacy = etree.tostring(xml_legacy).decode()
xml_updated = etree.tostring(xml_updated).decode()
self.assertEqual(xml_legacy, xml_updated)
def test_specific_10000047_with_new_crsecsturl_field(self):
xml_string = get_string("fixtures/large-test-file.xml")
root = ET.fromstring(xml_string)
accreditations = Accreditations(root)
kisaims = KisAims(root)
locations = Locations(root)
xml_string = get_string(
"fixtures/one_inst_one_course_with_crsecsturl.xml")
root = ET.fromstring(xml_string)
institution = get_first(root.iter("INSTITUTION"))
raw_inst_data = xmltodict.parse(
ET.tostring(institution))["INSTITUTION"]
ukprn = raw_inst_data["UKPRN"]
course = get_first(institution.findall("KISCOURSE"))
raw_course_data = xmltodict.parse(ET.tostring(course))["KISCOURSE"]
locids = get_locids(raw_course_data, ukprn)
expected_course_doc = json.loads(
get_string("fixtures/one_inst_one_course_with_crsecsturl.json"))
course_doc = get_course_doc(
accreditations,
locations,
locids,
raw_inst_data,
raw_course_data,
kisaims,
)
course_doc = remove_variable_elements(course_doc)
self.assertEqual(expected_course_doc, course_doc)
def _kantara_log_assertion_id(self, saml_response, ticket):
"""
Log the assertion id, which _might_ be required by Kantara.
:param saml_response: authn response as a compact XML string
:param ticket: Login process state
:type saml_response: str | unicode
:type ticket: SSOLoginData
:return: None
"""
printed = False
try:
parser = DefusedElementTree.DefusedXMLParser()
xml = DefusedElementTree.XML(str(saml_response), parser)
# For debugging, it is very useful to get the full SAML response pretty-printed in the logfile directly
self.logger.debug("Created AuthNResponse :\n\n{!s}\n\n".format(DefusedElementTree.tostring(xml)))
printed = True
attrs = xml.attrib
assertion = xml.find('{urn:oasis:names:tc:SAML:2.0:assertion}Assertion')
self.logger.info(
'{!s}: id={!s}, in_response_to={!s}, assertion_id={!s}'.format(
ticket.key, attrs['ID'], attrs['InResponseTo'], assertion.get('ID')
)
)
return DefusedElementTree.tostring(xml)
except Exception as exc:
self.logger.debug("Could not parse message as XML: {!r}".format(exc))
if not printed:
# Fall back to logging the whole response
self.logger.info("{!s}: authn response: {!s}".format(ticket.key, saml_response))
def _kantara_log_assertion_id(self, saml_response, ticket):
"""
Log the assertion id, which _might_ be required by Kantara.
:param saml_response: authn response as a compact XML string
:param ticket: Login process state
:type saml_response: str | unicode
:type ticket: SSOLoginData
:return: None
"""
printed = False
try:
parser = DefusedElementTree.DefusedXMLParser()
xml = DefusedElementTree.XML(str(saml_response), parser)
# For debugging, it is very useful to get the full SAML response pretty-printed in the logfile directly
self.logger.debug("Created AuthNResponse :\n\n{!s}\n\n".format(DefusedElementTree.tostring(xml)))
printed = True
attrs = xml.attrib
assertion = xml.find('{urn:oasis:names:tc:SAML:2.0:assertion}Assertion')
self.logger.info('{!s}: id={!s}, in_response_to={!s}, assertion_id={!s}'.format(
ticket.key, attrs['ID'], attrs['InResponseTo'], assertion.get('ID')))
return DefusedElementTree.tostring(xml)
except Exception as exc:
self.logger.debug("Could not parse message as XML: {!r}".format(exc))
if not printed:
# Fall back to logging the whole response
self.logger.info("{!s}: authn response: {!s}".format(ticket.key, saml_response))
def test_json_serializer_with_geoobject_adapter(self):
"""
test new serialization in contrast with legacy serializer to ensure compatibility
"""
from FreeTAKServer.controllers.serializers.api_adapters.api_adapters import GeoObjectAdapter
basejson = {
"attitude": "friendly",
"how": "nonCoT",
"name": "testing",
"latitude": "37.5677889",
"longitude": "12.345678",
"geoObject": "Ground",
"timeout": "120"
}
jsonobj = JsonController().serialize_geoobject_post(basejson)
simpleCoTObject = SendSimpleCoTController(jsonobj).getCoTObject()
xml_legacy = etree.fromstring(simpleCoTObject.xmlString)
xml_legacy.attrib['uid'] = '123'
xml_legacy.attrib['time'] = '24'
xml_legacy.attrib['start'] = '849'
xml_legacy.attrib['stale'] = '3333'
newObject = GeoObjectAdapter().from_api_to_fts_object(basejson)
xml_updated = etree.fromstring(newObject.xmlString)
xml_updated.attrib['uid'] = '123'
xml_updated.attrib['time'] = '24'
xml_updated.attrib['start'] = '849'
xml_updated.attrib['stale'] = '3333'
xml_legacy = etree.tostring(xml_legacy).decode()
xml_updated = etree.tostring(xml_updated).decode()
self.assertEqual(xml_legacy, xml_updated)
def test_with_large_file(self):
xml_string = get_string("fixtures/large-test-file.xml")
root = ET.fromstring(xml_string)
for institution in root.iter("INSTITUTION"):
raw_inst_data = xmltodict.parse(
ET.tostring(institution))["INSTITUTION"]
for course in institution.findall("KISCOURSE"):
raw_course_data = xmltodict.parse(
ET.tostring(course))["KISCOURSE"]
need_nhs_nss(raw_course_data)
def create_course_docs(xml_string):
"""Parse HESA XML passed in and create JSON course docs in Cosmos DB."""
# TODO Investigate writing docs to CosmosDB in bulk to speed things up.
cosmosdb_client = utils.get_cosmos_client()
logging.info(
"adding ukrlp data into memory ahead of building course documents"
)
enricher = UkRlpCourseEnricher()
logging.info(
"adding subject data into memory ahead of building course documents"
)
subject_enricher = SubjectCourseEnricher()
collection_link = utils.get_collection_link(
"AzureCosmosDbDatabaseId", "AzureCosmosDbCoursesCollectionId"
)
# Import the XML dataset
root = ET.fromstring(xml_string)
# Import accreditations, common, kisaims and location nodes
accreditations = Accreditations(root)
kisaims = KisAims(root)
locations = Locations(root)
course_count = 0
for institution in root.iter("INSTITUTION"):
raw_inst_data = xmltodict.parse(ET.tostring(institution))[
"INSTITUTION"
]
ukprn = raw_inst_data["UKPRN"]
for course in institution.findall("KISCOURSE"):
raw_course_data = xmltodict.parse(ET.tostring(course))["KISCOURSE"]
locids = get_locids(raw_course_data, ukprn)
course_doc = get_course_doc(
accreditations,
locations,
locids,
raw_inst_data,
raw_course_data,
kisaims,
)
enricher.enrich_course(course_doc)
subject_enricher.enrich_course(course_doc)
cosmosdb_client.CreateItem(collection_link, course_doc)
course_count += 1
logging.info(f"Processed {course_count} courses")
def specificPackage():
from defusedxml import ElementTree as etree
from os import listdir
if request.method == 'GET' and request.args.get('uid') != None:
data = request.data
taskuid = request.args.get('uid')
for file in listdir(MainConfig.ExCheckChecklistFilePath):
xml = etree.parse(
str(
PurePath(Path(MainConfig.ExCheckChecklistFilePath),
Path(file)))).getroot()
tasks = xml.find('checklistTasks')
for task in tasks:
uid = task.find('uid')
if str(uid.text) == str(taskuid):
return etree.tostring(task)
else:
pass
for file in listdir(MainConfig.ExCheckFilePath):
xml = etree.parse(
str(PurePath(Path(MainConfig.ExCheckFilePath),
Path(file)))).getroot()
if xml.find("checklistDetails").find('uid').text == str(taskuid):
return etree.tostring(xml)
else:
hash = request.args.get('hash')
import hashlib
if os.path.exists(str(PurePath(Path(dp_directory), Path(hash)))):
logger.info('marti sync content triggerd')
app.logger.debug(str(PurePath(Path(dp_directory), Path(hash))))
file_list = os.listdir(
str(PurePath(Path(dp_directory), Path(hash))))
app.logger.debug(
PurePath(Path(const.DATAPACKAGEFOLDER), Path(hash),
Path(file_list[0])))
path = PurePath(dp_directory, str(hash), file_list[0])
app.logger.debug(str(path))
return send_file(str(path))
else:
obj = dbController.query_ExCheck(verbose=True,
query=f'hash == "{hash}"')
data = etree.parse(
str(
PurePath(Path(MainConfig.ExCheckFilePath),
Path(obj[0].data.filename))))
data.getroot().find('checklistTasks').find("checklistTask").find(
"uid").text = data.getroot().find('checklistTasks').find(
"checklistTask").find("checklistUid").text
output = etree.tostring(data, pretty_print=False)
return output
def test_with_large_file(self):
"""Initial smoke test"""
xml_string = get_string("fixtures/large-test-file.xml")
root = ET.fromstring(xml_string)
for institution in root.iter("INSTITUTION"):
raw_inst_data = xmltodict.parse(
ET.tostring(institution))["INSTITUTION"]
country_code = get_country(raw_inst_data)["code"]
for course in institution.findall("KISCOURSE"):
raw_course_data = xmltodict.parse(
ET.tostring(course))["KISCOURSE"]
leo = Leo(country_code)
leo.get_stats(raw_course_data)
def main(self):
from defusedxml import ElementTree as etree
import time
while True:
time.sleep(0.1)
command = self.receive_command_data(self.pipe)
if command:
try:
self.m_SendConnectionHandler.Handle(self, command)
except Exception as e:
logger.debug("exception in command chain " + str(e))
else:
pass
data = self.receive_data_from_federate(1)
if data:
for protobuf_object in data:
# TODO: clean all of this up as it's just a PoC
try:
detail = etree.fromstring(protobuf_object.event.other)
protobuf_object.event.other = ''
fts_obj = ProtobufSerializer(
).from_format_to_fts_object(protobuf_object,
Event.FederatedCoT())
specific_obj = SendOther()
event = XmlSerializer().from_fts_object_to_format(
fts_obj)
xmlstring = event
xmlstring.find('detail').remove(
xmlstring.find('detail').find('remarks'))
xmlstring.find('detail').extend(
[child for child in detail])
# specific_obj.xmlString = etree.tostring(xmlstring)
print(etree.tostring(xmlstring))
specific_obj.xmlString = etree.tostring(xmlstring)
self.pipe.send(specific_obj)
except Exception as e:
pass
"""if isinstance(SpecificCoTObj, SendOtherController):
detail = protobuf_object.event.other
protobuf_object.event.other = ''
fts_obj = ProtobufSerializer().from_format_to_fts_object(protobuf_object, Event.Other())
protobuf_object.event.other = detail
SpecificCoTObj.object = fts_obj
SpecificCoTObj.Object =
else:
fts_obj = ProtobufSerializer().from_format_to_fts_object(protobuf_object, SpecificCoTObj().object)
self.pipe.send(data)"""
else:
pass
def test_xml_serialization(self):
"""
test new serialization in contrast with legacy serializer to ensure compatibility
"""
from FreeTAKServer.model.FTSModel.Event import Event
# xmlstring = '<event version="2.0" uid="GeoChat.ANDROID-359975090666199.FEATHER.27d8ef23-8578-4cb4-8f53-02f5dc150cd2" type="b-t-f" how="h-g-i-g-o" start="2021-01-03T19:01:35.472Z" time="2021-01-03T19:01:35.472Z" stale="2021-01-04T19:01:35.472Z"><detail><__chat id="S-1-5-21-2720623347-3037847324-4167270909-1002" parent="RootContactGroup" chatroom="FEATHER" groupOwner="false"><chatgrp uid0="ANDROID-359975090666199" uid1="S-1-5-21-2720623347-3037847324-4167270909-1002" id="S-1-5-21-2720623347-3037847324-4167270909-1002"/></__chat><link uid="ANDROID-359975090666199" relation="p-p" type="a-f-G-E-V-A"/><remarks time="2021-01-03T19:01:35.472Z" source="BAO.F.ATAK.ANDROID-359975090666199" to="S-1-5-21-2720623347-3037847324-4167270909-1002">at VDO</remarks><__serverdestination destinations="192.168.2.66:4242:tcp:ANDROID-359975090666199"/><marti><dest callsign = "WOLF"/><dest callsign="GALLOP"/><dest callsign="FEATHER"/></marti></detail><point le="9999999.0" ce="3.2" hae="22.958679722315807" lon="-66.10803" lat="43.855711"/></event>'
xmlstring = '<event version="2.0" uid="115e1a1a-9947-4531-801c-40215fa4888c" type="b-m-r" time="2021-03-17T21:53:08.07Z" start="2021-03-17T21:53:08.07Z" stale="2021-03-24T21:53:08.07Z" how="h-e"><point lat="0" lon="0" hae="0" ce="9999999" le="9999999" /><detail><contact callsign="Eliopoli/BOP" /><link uid="4eaf4026-8db6-48ef-83c2-063dd0120339" type="b-m-p-w" point="44.1644630, -65.7995800,85.84967474" callsign="Eliopoli/BOP SP" relation="c" remarks="" /><link uid="42882562-9dc6-481b-9c89-9a5ba96fbf57" type="b-m-p-w" point="43.9676310, -66.1266740,16.80836837" callsign="Eliopoli" relation="c" remarks="" /><link_attr color="-1" type="b-m-r" method="Driving" direction="Infil" routetype="Primary" order="Ascending Check Points" /></detail></event>'
# fts_obj = XmlSerializer().from_format_to_fts_object(xmlstring, Event.GeoChat())
fts_obj = XmlSerializer().from_format_to_fts_object(xmlstring, Event.Route())
obj = XmlSerializer().from_fts_object_to_format(fts_obj)
print(etree.tostring(obj).decode())
# legacyftsobj = XMLCoTController().serialize_CoT_to_model(Event.GeoChat(), etree.fromstring(xmlstring))
legacy_string = XMLCoTController().serialize_model_to_CoT(fts_obj)
self.assertEqual(legacy_string.decode(), etree.tostring(obj).decode())
def test_descriptor_repr(self) -> None:
"""Test descriptor's repr can convert back to an equivalent descriptorb."""
# pylint: disable=import-outside-toplevel
from didl_lite.didl_lite import Descriptor
descriptor = Descriptor(id="1",
name_space="ns",
type="type",
text="Text")
descriptor_repr = repr(descriptor)
descriptor_remade = eval(descriptor_repr) # pylint: disable=eval-used
assert ET.tostring(descriptor.to_xml()) == ET.tostring(
descriptor_remade.to_xml())
def from_format_to_fts_object(self, object: type, FTSObject: Event) -> Event:
for descriptor in object.event.DESCRIPTOR.fields:
attribute_name = descriptor.name
if attribute_name in self.attribute_name_mapping:
attribute_name = self.attribute_name_mapping[attribute_name]
if hasattr(FTSObject.detail, 'marti'):
from FreeTAKServer.model.FTSModel.Dest import Dest
for callsign in object.event.ptpCallsigns:
newdest = Dest()
newdest.setcallsign(callsign)
FTSObject.detail.marti.setdest(newdest)
elif attribute_name != 'other' and attribute_name != '':
setters = self._get_fts_object_var_setter(FTSObject, attribute_name)
setter = self._get_method_in_method_list(setters, 'event')
if attribute_name == 'time' or attribute_name == 'start' or attribute_name == 'stale':
import datetime
attribute = getattr(object.event, descriptor.name)
try:
setter(datetime.datetime.strftime(
datetime.datetime.strptime(str(datetime.datetime.fromtimestamp(float(attribute) / 1000.0)),
"%Y-%m-%d %H:%M:%S.%f"), "%Y-%m-%dT%H:%M:%S.%fZ"))
continue
except Exception as e:
setter(datetime.datetime.strftime(
datetime.datetime.strptime(str(datetime.datetime.fromtimestamp(float(attribute) / 1000.0)),
"%Y-%m-%d %H:%M:%S"), "%Y-%m-%dT%H:%M:%S.%fZ"))
continue
else:
setter(getattr(object.event, descriptor.name))
elif attribute_name == 'other' and object.event.other != '':
from defusedxml import ElementTree as etree
from lxml.etree import Element
xmldetail = etree.fromstring(object.event.other)
xmldetail.remove(xmldetail.find('_flow-tags_'))
if xmldetail.find('remarks'):
XmlSerializer().from_format_to_fts_object(etree.tostring(xmldetail).decode(), FTSObject.detail)
else:
xmldetail.append(Element('remarks'))
xmldetail.find('remarks').text = 'From federation '
XmlSerializer().from_format_to_fts_object(etree.tostring(xmldetail).decode(), FTSObject.detail)
return FTSObject
def filter_out(filename, tmp_filename):
"""Filter out generated code and Dezyne code."""
with open(filename, "r", encoding="utf-8") as input_file:
doc = ElementTree.parse(input_file)
for elem in doc.xpath("//*/Project"):
if "Proxy" in elem.attrib["Name"]:
parent = elem.getparent()
parent.remove(elem)
if "Dezyne" in elem.attrib["Name"]:
parent = elem.getparent()
parent.remove(elem)
print(ElementTree.tostring(doc))
with open(tmp_filename, "w", encoding="utf-8") as tmp_file:
tmp_file.write(str(ElementTree.tostring(doc, encoding="unicode")))
def create_ukrlp_lookups(self):
"""Parse HESA XML and create JSON lookup table for UKRLP data."""
root = ET.fromstring(self.xml_string)
for institution in root.iter("INSTITUTION"):
raw_inst_data = xmltodict.parse(
ET.tostring(institution))["INSTITUTION"]
ukprn = raw_inst_data.get("UKPRN")
pubukprn = raw_inst_data.get("PUBUKPRN")
if ukprn and not self.entry_exists(ukprn):
if self.create_ukrlp_lookup(ukprn):
self.lookups_created.append(ukprn)
if pubukprn and not self.entry_exists(pubukprn):
if self.create_ukrlp_lookup(pubukprn):
self.lookups_created.append(ukprn)
logging.info(f"lookups_created = {len(self.lookups_created)}")
if self.ukrlp_no_info_list:
logging.info(f"UKRLP did not return info for the following "
f"{len(self.ukrlp_no_info_list)} ukprn(s)")
for ukprn in self.ukrlp_no_info_list:
logging.info(f"{ukprn}")
logging.info(
f"DB entries existed for {len(self.db_entries_list)} ukprns tried")
def activechecklists():
from os import listdir
from FreeTAKServer.model.FTSModel.Checklists import Checklists
from FreeTAKServer.model.FTSModel.Checklist import Checklist
from lxml.etree import Element
from lxml import etree
checklists = Checklists.Checklist()
rootxml = Element('checklists')
for file in listdir(MainConfig.ExCheckChecklistFilePath):
try:
checklist = Element('checklist')
xmldetails = etree.parse(
str(
PurePath(Path(MainConfig.ExCheckChecklistFilePath),
Path(file)))).getroot().find('checklistDetails')
checklist.append(xmldetails)
checklist.append(Element('checklistColumns'))
checklist.append(Element('checklistTasks'))
rootxml.append(checklist)
except Exception as e:
print(e)
xml = etree.tostring(rootxml)
return xml
def test_xml_serialization(self):
"""
test new serialization in contrast with legacy serializer to ensure compatibility
"""
from FreeTAKServer.model.FTSModel.Event import Event
# xmlstring = '<event version="2.0" uid="GeoChat.ANDROID-359975090666199.FEATHER.27d8ef23-8578-4cb4-8f53-02f5dc150cd2" type="b-t-f" how="h-g-i-g-o" start="2021-01-03T19:01:35.472Z" time="2021-01-03T19:01:35.472Z" stale="2021-01-04T19:01:35.472Z"><detail><__chat id="S-1-5-21-2720623347-3037847324-4167270909-1002" parent="RootContactGroup" chatroom="FEATHER" groupOwner="false"><chatgrp uid0="ANDROID-359975090666199" uid1="S-1-5-21-2720623347-3037847324-4167270909-1002" id="S-1-5-21-2720623347-3037847324-4167270909-1002"/></__chat><link uid="ANDROID-359975090666199" relation="p-p" type="a-f-G-E-V-A"/><remarks time="2021-01-03T19:01:35.472Z" source="BAO.F.ATAK.ANDROID-359975090666199" to="S-1-5-21-2720623347-3037847324-4167270909-1002">at VDO</remarks><__serverdestination destinations="192.168.2.66:4242:tcp:ANDROID-359975090666199"/><marti><dest callsign = "WOLF"/><dest callsign="GALLOP"/><dest callsign="FEATHER"/></marti></detail><point le="9999999.0" ce="3.2" hae="22.958679722315807" lon="-66.10803" lat="43.855711"/></event>'
xmlstring = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><event version="2.0" uid="03Z0170972" type="a-f-A-M-H-Q" time="2021-06-17T19:08:19.699Z" start="2021-06-17T19:08:19.699Z" stale="2021-06-18T19:08:23.199Z" how="m-g"><point lat="43.85566678637877" lon="-66.10800161079035" hae="0.11165844181401496" ce="0.0" le="0.0" /><detail> <contact callsign="UAS-NOVA" /><sensor elevation="0.0" vfov="66.0" north="226.70000076293945" roll="0.0" range="300" azimuth="46.0" model="Phantom 3 Advanced Camera" fov="81.0" type="r-e" version="0.6" /></detail></event>'
# fts_obj = XmlSerializer().from_format_to_fts_object(xmlstring, Event.GeoChat())
fts_obj = XmlSerializer().from_format_to_fts_object(
xmlstring, Event.DroneSensor())
obj = XmlSerializer().from_fts_object_to_format(fts_obj)
print(etree.tostring(obj).decode())
# legacyftsobj = XMLCoTController().serialize_CoT_to_model(Event.GeoChat(), etree.fromstring(xmlstring))
legacy_string = XMLCoTController().serialize_model_to_CoT(fts_obj)
self.assertEqual(legacy_string.decode(), etree.tostring(obj).decode())
def maybe_xml_to_string(message, logger=None):
"""
Try to parse message as an XML string, and then return it pretty-printed.
If message couldn't be parsed, return string representation of it instead.
This is used to (debug-)log SAML requests/responses in a readable way.
:param message: XML string typically
:param logger: logging logger
:return: something ready for logging
:rtype: string
"""
if isinstance(message, six.binary_type):
# message is returned as binary from pysaml2 in python3
message = message.decode('utf-8')
message = str(message)
try:
from defusedxml import ElementTree as DefusedElementTree
parser = DefusedElementTree.DefusedXMLParser()
xml = DefusedElementTree.XML(message, parser)
return DefusedElementTree.tostring(xml)
except Exception as exc:
if logger is not None:
logger.debug(
"Could not parse message of type {!r} as XML: {!r}".format(
type(message), exc))
return message
def append_security(xml_text, security_statement, ns=FHIR_NAMESPACE):
"""
pass in an xml text block
append a security xml text block to it
xml_text needs to be the "rest" segment
:param xml_text:
:param security_statement:
:return:
"""
root = xml_to_dom(xml_text)
element_rest = root.findall('{%s}rest' % ns_string(ns), ns)
sec_dom = xml_to_dom(security_statement)
for rest_item in element_rest:
rest_item.append(sec_dom)
# Add sec_dom to "rest" section of dom
back_to_xml = ET.tostring(root).decode("utf-8")
# print("\nUpdated DOM:%s" % back_to_xml)
return back_to_xml
def _replace_at(self, index):
"""
Replaces the text instance at the given index with braces for use with string formatting. String formatting in
replace_text() allows for XML characters, such as &, to be set within the text.
:param index: index number
:return: replacement string with format braces
"""
replacement = ""
# copy over
root = deepcopy(self._root)
# iterate
current = -1
for element in root.iter():
# count
if element.text:
current += 1
# replace at index
if current == index:
element.text = '{}'
replacement = ElementTree.tostring(root,
encoding="unicode")
break
# return
return replacement
def convert(folder):
root = Element('SokobanLevels')
title = SubElement(root, 'Title')
title.text = folder.split('/')[-1]
desc = SubElement(root, 'Description')
desc.text = folder.split('/')[-1]
collection = SubElement(root, 'LevelCollection')
i = 1
def natural_keys(text):
return [int(c) if c.isdigit() else c for c in re.split(r'(\d+)', text)]
for file in sorted(os.listdir(folder), key=natural_keys):
if not file.endswith('.txt'):
continue
level = SubElement(collection, 'Level', {'Id': str(i)})
with open('/'.join([folder, file]), 'r') as f:
for r in f.readlines():
line = SubElement(level, 'L')
line.text = r[:-1].rstrip()
i += 1
s = ElementTree.tostring(root, encoding='utf-8')
s = b'<!DOCTYPE SokobanLevels SYSTEM "SokobanLev.dtd">' + s
parsed = minidom.parseString(s)
xml = parsed.toprettyxml(indent=' ', encoding='utf-8', newl='\n')
with open(f'{folder}.slc', 'wb') as f:
f.write(xml)
def render(self, tree):
property_template = jinja2.Environment(
autoescape=True, loader=jinja2.FileSystemLoader(
property_template_dir), trim_blocks='true')\
.get_template(property_template_file_name)
subcategory = ''
if tree.find('subcategory') is not None:
subcategory = self.get_element_text(tree.find('subcategory'))
description = ''
if tree.find('description') is not None:
description = self.get_element_text(tree.find('description')).replace('"', '\\"')
syntax = ''
if tree.find('syntax') is not None:
syntax = etree.tostring(tree.find('syntax'))[9:-12].decode("utf-8")\
.replace('"', '\\"').replace('\n', '\\n')
print(str(syntax))
parameters = []
for parameter in tree.iterfind('parameter'):
label = self.get_element_text(parameter.find('label'))
param_description = ''
if parameter.find('description') is not None:
param_description = self.get_element_text(parameter.find('description'))\
.replace('"', '\\"').replace('\n', ' ')
parameters.append({'label':label, 'description':param_description})
# TODO: Support method
methods = []
for method in tree.iterfind('method'):
label = self.get_element_text(method.find('label'))
param_description = method.find('description')
# ref = create_ref_link(self.get_element_text(method.find('ref')))
# self.methods.append({'label':label, 'description':param_description, 'ref':ref})
constructors = []
for constructor in tree.iterfind('constructor'):
constructors.append(self.get_element_text(constructor))
relateds = []
for related in tree.iterfind('related'):
relateds.append(self.get_element_text(related))
item = {
'category': self.get_element_text(tree.find('category')),
'subcategory': subcategory,
'description': description,
'syntax': syntax,
'parameters': parameters,
'methods': methods,
'constructors': constructors,
'related': relateds
# 'category': self.get_element_text(tree.find('category')),
# 'category': self.get_element_text(tree.find('category')),
# 'category': self.get_element_text(tree.find('category')),
# 'category': self.get_element_text(tree.find('category')),
# 'category': self.get_element_text(tree.find('category'))
}
return property_template.render(item=item)
def _do_scheme(self):
scheme = Scheme(self.title)
scheme.description = self.description
scheme.use_external_validation = self.use_external_validation
scheme.streaming_mode = Scheme.streaming_mode_xml
scheme.use_single_instance = self.use_single_instance
for argument in self.extra_arguments():
name = argument["name"]
title = argument.get("title", None)
description = argument.get("description", None)
validation = argument.get("validation", None)
data_type = argument.get("data_type", Argument.data_type_string)
required_on_edit = argument.get("required_on_edit", False)
required_on_create = argument.get("required_on_create", False)
scheme.add_argument(
Argument(
name,
title=title,
description=description,
validation=validation,
data_type=data_type,
required_on_edit=required_on_edit,
required_on_create=required_on_create,
))
return defused_et.tostring(scheme.to_xml(), encoding="unicode")
def send_data_to_clients(self, data):
from defusedxml import ElementTree as etree
try:
if self.federates:
xmlstring = data.xmlString
detail = etree.fromstring(xmlstring).find('detail')
if detail:
protobuf = ProtobufSerializer().from_fts_object_to_format(
data.modelObject)
try:
protobuf.event.other = etree.tostring(detail)
protobufstring = protobuf.SerializeToString()
header = self._generate_header(len(protobufstring))
protobufstring = header + protobufstring
print(protobufstring)
except Exception as e:
logger.warning("creating protobuf message failed " +
str(e))
return None
for client in self.federates.values():
client.conn.send(protobufstring)
else:
return None
else:
return None
except Exception as e:
import traceback
trace = traceback.format_exc()
logger.warning("sending data to federates failed " + str(e))
def maybe_xml_to_string(message, logger=None):
"""
Try to parse message as an XML string, and then return it pretty-printed.
If message couldn't be parsed, return string representation of it instead.
This is used to (debug-)log SAML requests/responses in a readable way.
:param message: XML string typically
:param logger: logging logger
:return: something ready for logging
:rtype: string
"""
if isinstance(message, six.binary_type):
# message is returned as binary from pysaml2 in python3
message = message.decode('utf-8')
message = str(message)
try:
from defusedxml import ElementTree as DefusedElementTree
parser = DefusedElementTree.DefusedXMLParser()
xml = DefusedElementTree.XML(message, parser)
return DefusedElementTree.tostring(xml)
except Exception as exc:
if logger is not None:
logger.debug("Could not parse message of type {!r} as XML: {!r}".format(type(message), exc))
return message
def __init__(self, RawCoT=None, addToDB=MainConfig.SaveCoTToDB):
if type(RawCoT != bytes):
pass
else:
RawCoT.xmlString.encode()
try:
tempObject = super().Event.Other()
self.object = SendOther()
if RawCoT is not None:
xml = RawCoT.xmlString
RawCoT.xmlString = etree.tostring(self.filter_CoT(xml))
self.fill_object(self.object,
tempObject,
RawCoT,
addToDB=addToDB)
try:
object = self.getObject()
except Exception as e:
logger.error(
"there has been an exception getting object " + str(e))
self.Object.setXmlString(xml)
else:
pass
except Exception as e:
logger.error("there has been an exception in the creation of an"
"Other object " + str(e))
def add_anchor(html, anchor_link_text=u'¶'):
"""Add an id and an anchor-link to an html header
For use on markdown headings
"""
try:
h = ElementTree.fromstring(py3compat.cast_bytes_py2(html, encoding='utf-8'))
except Exception:
# failed to parse, just return it unmodified
return html
link = _convert_header_id(html2text(h))
h.set('id', link)
a = Element("a", {"class": "anchor-link", "href": "#" + link})
try:
# Test if the anchor link text is HTML (e.g. an image)
a.append(ElementTree.fromstring(anchor_link_text))
except Exception:
# If we fail to parse, assume we've just got regular text
a.text = anchor_link_text
h.append(a)
# Known issue of Python3.x, ElementTree.tostring() returns a byte string
# instead of a text string. See issue http://bugs.python.org/issue10942
# Workaround is to make sure the bytes are casted to a string.
return py3compat.decode(ElementTree.tostring(h), 'utf-8')
def genSaveGameInfo(savegame_file, read_data=False):
farmer = createFarmer(findPlayer(savegame_file, read_data))
savegameinfo = ET.tostring(farmer, encoding='UTF-8',
method='xml').strip(str_or_bytes('\r\n'))
savegameinfo = savegameinfo.replace(str_or_bytes(target_namespaces),
str_or_bytes(required_namespaces), 1)
return savegameinfo
def main() -> None:
fw = Firewall("10.0.1.25", "admin", "Pal0Alt0!")
for index in range(1, 5):
addobj = fw.add(AddressObject(f"My-new-object-{index}", "1.1.1.1"))
addobj.create()
result = fw.op("show system info")
print(ET.tostring(result))
def list_pages(self, url, max_items=200, offset=0):
response = self.get_from_hbogo(
url + self.LANGUAGE_CODE + "&max=" + str(max_items) + "&offset=" +
str(offset), 'xml')
if response is False:
return
count = 0
for item in response.findall('.//item'):
count += 1
item_link = item.find('link').text
if item_link:
if self.lograwdata:
self.log(ET.tostring(item, encoding='utf8'))
item_type = py2_encode(
item.find('clearleap:itemType',
namespaces=self.NAMESPACES).text)
if item_type != 'media':
self.addDir(item)
elif item_type == 'media':
self.addLink(item, HbogoConstants.ACTION_PLAY)
else:
self.log('Unknown item type: ' + item_type)
self.log('List pages total items: ' + str(count))
if count == max_items:
self.log('List pages calling next page... max: ' + str(max_items) +
' offset: ' + str(offset + max_items))
self.list_pages(url, max_items, offset + max_items)
def genSaveGameInfo(savegame_file):
farmer = createFarmer(findPlayer(savegame_file))
savegameinfo = ET.tostring(farmer, encoding='UTF-8',
method='xml').strip('\r\n')
savegameinfo = savegameinfo.replace(target_namespaces, required_namespaces,
1)
return savegameinfo
def __init__(self, root):
"""Build the locations lookup table
The key is comprised of two parts:
UKPRN
LOCID
If LOCATION.LOCUKPRN is present, we use that for the UKPRN part
otherwise we use LOCATION.UKPRN for the UKPRN part.
The LOCID part is the LOCATION.LOCID
"""
self.lookup_dict = {}
for location in root.iter("LOCATION"):
try:
raw_location_data = xmltodict.parse(
ET.tostring(location))["LOCATION"]
ukprn = raw_location_data["UKPRN"]
if "LOCUKPRN" in raw_location_data:
ukprn = raw_location_data["LOCUKPRN"]
lockey = f"{ukprn}{raw_location_data['LOCID']}"
self.lookup_dict[lockey] = raw_location_data
except Exception as e:
if hasattr(e, 'message'):
exception_text = e.message
else:
exception_text = e
def _pack_req(self, element):
"""Pack an etree Element.
@param element : a etree.Element
@returns : a string, wrapping element within the request tags
"""
return self.soap_envelope.format(
codecs.decode(etree.tostring(element), 'utf-8')).encode('utf-8')
def get_site_devices(self, site_id):
"""Return a list of devices in a site."""
xml_string = (
'<SiteDeviceListingRequest session-id = "%s" \
site-id = "%s"></SiteDeviceListingRequest>'
% (self.session_id, site_id)
)
xml_response = self.api_request(xml_string)
print(ET.tostring(xml_response, encoding="ascii", method="xml"))
def __init__(self, filename, test):
bug_patterns = dict()
dupes = dict()
SEVERITY = {
'1': 'High',
'2': 'Medium',
'3': 'Low'
}
tree = ET.parse(filename)
root = tree.getroot()
for pattern in root.findall('BugPattern'):
plain_pattern = re.sub(r'<[b-z/]*?>|<a|</a>|href=', '', ET.tostring(pattern.find('Details'), method='text'))
bug_patterns[pattern.get('type')] = plain_pattern
for bug in root.findall('BugInstance'):
desc = ''
for message in bug.itertext():
desc += message
dupe_key = bug.get('instanceHash')
title = bug.find('ShortMessage').text
cwe = bug.get('cweid', default=0)
severity = SEVERITY[bug.get('priority')]
description = desc
mitigation = bug_patterns[bug.get('type')]
impact = 'N/A'
references = 'N/A'
if dupe_key in dupes:
finding = dupes[dupe_key]
else:
finding = Finding(
title=title,
cwe=cwe,
severity=severity,
description=description,
mitigation=mitigation,
impact=impact,
references=references,
test=test,
active=False,
verified=False,
numerical_severity=Finding.get_numerical_severity(severity),
static_finding=True
)
dupes[dupe_key] = finding
self.items = dupes.values()
def printable_data(self, pretty=True):
string = ET.tostring(self.data, 'utf-8').decode()
if pretty:
string = minidom.parseString(string).toprettyxml(indent=' ').split('\n', 1)[1]
return string
def _get_raw_xml(data, path):
xml = data.find(path)
if xml is not None:
return ElementTree.tostring(xml, encoding="utf-8")
else:
return ""
def genSaveGameInfo(savegame_file,read_data=False):
farmer = createFarmer(findPlayer(savegame_file,read_data))
savegameinfo = ET.tostring(farmer, encoding='UTF-8', method='xml').strip('\r\n')
savegameinfo = savegameinfo.replace(target_namespaces,required_namespaces,1)
return savegameinfo
def as_file(svg_xml):
"""Converts the xml object into a django ContentFile"""
return ContentFile(ElementTree.tostring(svg_xml))