def parse_response(self,
action_name,
content,
encoding=None,
envelope_attrib=None,
typed=None):
"""
Parses a response from the server with the given action name and content.
"""
register_namespace('', None)
if encoding is None:
encoding = self._encoding
if envelope_attrib is None:
envelope_attrib = self._envelope_attrib
if typed is None:
typed = self._typed
try:
docNode = xml_fromstring(content)
except ParseError:
# Try removing any extra XML declarations in case there are more than one.
# This sometimes happens when a device sends its own XML config files.
content = remove_extraneous_xml_declarations(content)
docNode = xml_fromstring(content)
except ValueError:
# This can occur when requests returns a `str` (unicode) but there's also an XML
# declaration, which lxml doesn't like.
docNode = xml_fromstring(content.encode('utf8'))
resp_body = None
if typed:
resp_body = docNode.find(".//{%s}%sResponse" %
(typed, action_name))
else:
resp_body = docNode.find(".//%sResponse" % action_name)
if resp_body is None:
msg = (
'Returned XML did not include an element which matches namespace %r and tag name'
' \'%sResponse\'.' % (typed, action_name))
print(msg + '\n' + xml_tostring(
docNode, short_empty_elements=False).decode('utf8'))
raise SOAPProtocolError(msg)
# Sometimes devices return XML strings as their argument values without escaping them with
# CDATA. This checks to see if the argument has been parsed as XML and un-parses it if so.
resp_dict = {}
for arg in resp_body.getchildren():
children = arg.getchildren()
if children:
resp_dict[arg.tag] = "\n".join(
xml_tostring(x) for x in children)
else:
if arg.text is None:
resp_dict[arg.tag] = ""
else:
resp_dict[arg.tag] = arg.text
return resp_dict
def stat(self, cp_code=None, path=''):
params = {}
params['action'] = Actions.STAT
response, status = self.send(cp_code, path, params)
if status == 200:
try:
stats = xml_fromstring(response)
element = stats.find('file')
return self.__dir_action_entry(element, path)
except ExpatError, parse_error:
raise AkamaiResponseMalformedException(str(parse_error))
def du(self, cp_code=None, path=None, params=None):
params = params or {}
params['action'] = Actions.DU
# Making the request
response, status = self.send(cp_code, path, params)
if status == 200:
try:
tree = xml_fromstring(response)
info = tree.find('du-info').attrib
return {'files': int(info['files']), 'bytes': int(info['bytes'])}
except ParseError, parse_error:
raise AkamaiResponseMalformedException(str(parse_error))
def get_data(self,
unique_data_ids=None,
sample=False,
output_type='csv',
**kwargs):
"""
Returns a JSON object of the entire data set.
"""
data_json = None
db = kwargs.get('db', None)
if unique_data_ids is None:
unique_data_ids = self._available_unique_data_ids
for u in unique_data_ids:
if (u not in self._available_unique_data_ids):
logger.info(
" The unique_data_id '{}' is not supported by the DhcdApiConn"
.format(u))
else:
result = self.get(self._urls[u], params=self._params[u])
if result.status_code != 200:
err = "An error occurred during request: status {0}"
logger.exception(err.format(result.status_code))
continue
data_xml_root = xml_fromstring(result.text)
data_xml_records = data_xml_root.findall('record')
data_json = xml_to_json.data(data_xml_root)
results = [
DhcdResult({e.tag: e.text
for e in list(r)}, self._fields[u]).data
for r in data_xml_records
]
self.result_to_csv(self._fields[u], results,
self.output_paths[u])
#Convert to format expected by database
if u == 'dhcd_dfd_properties':
self.create_project_subsidy_csv('dhcd_dfd_properties',
PROJECT_FIELDS_MAP,
SUBSIDY_FIELDS_MAP,
PROJECT_ADDRE_FIELDS_MAP,
db)
def dir(self, cp_code=None, path=None, params=None):
params = params or {}
params['action'] = Actions.DIR
# Making the request
response, status = self.send(cp_code, path, params)
# Transform the element into a well know dict format
# It will look for these attributes:
# 'type', 'name', 'mtime' and, if present, also include the
# 'size', 'target' attributes
def dir_action_entry(element):
attribs = dict(**element.attrib)
entry = {
'type': attribs['type'],
'name': attribs['name'],
'mtime': attribs['mtime'],
'path': path_join(path, attribs['name']) if path else attribs['name']
}
# Optional keys
for k in ('size', 'target', ):
try:
entry[k] = attribs[k]
except KeyError:
pass
# Integer keys
for k in ('size', 'mtime', ):
try:
entry[k] = int(entry[k])
except (KeyError, ValueError, ):
pass
# Datetime keys
for k in ('mtime', ):
try:
entry[k] = datetime.fromtimestamp(entry[k])
except (KeyError, ValueError, ):
pass
return entry
if status == 200:
try:
tree = xml_fromstring(response)
return [dir_action_entry(element) for element in tree.findall('file')]
except ParseError, parse_error:
raise AkamaiResponseMalformedException(str(parse_error))
def dir(self, cp_code=None, path=None, params=None):
params = params or {}
params['action'] = Actions.DIR
# Making the request
response, status = self.send(cp_code, path, params)
if status == 200:
try:
tree = xml_fromstring(response)
return [
self.__dir_action_entry(element, path)
for element in tree.findall('file')
]
except ExpatError, parse_error:
raise AkamaiResponseMalformedException(str(parse_error))
def du(self, cp_code=None, path=None, params=None):
params = params or {}
params['action'] = Actions.DU
# Making the request
response, status = self.send(cp_code, path, params)
if status == 200:
try:
tree = xml_fromstring(response)
info = tree.find('du-info').attrib
return {
'files': int(info['files']),
'bytes': int(info['bytes'])
}
except ParseError, parse_error:
raise AkamaiResponseMalformedException(str(parse_error))
def device_description_load(location: str) -> Union[ElementTree, None]:
"""
Gets the description document from the specified URL and loads the contents into an XML ElementTree.
:param location: The url of where to get the device description.
:returns: The XML ElementTree for the XML content from the device description or None.
"""
docTree = None
resp = requests.get(location)
if resp.status_code == 200:
xmlcontent = resp.content
docTree = ElementTree(xml_fromstring(xmlcontent))
return docTree
def _locked_process_full_service_description(self, sdurl: str):
"""
Downloads and processes the service description document.
:param sdurl: The url of the service description document to process.
"""
svcdesc = None
resp = requests.get(sdurl)
if resp.status_code == 200:
svcdesc = {}
namespaces = {"": UPNP_SERVICE1_NAMESPACE}
try:
xml_content = resp.text
descDoc = xml_fromstring(xml_content)
specVersionNode = descDoc.find("specVersion",
namespaces=namespaces)
verInfo = self._locked_process_node_spec_version(
specVersionNode, namespaces=namespaces)
svcdesc["specVersion"] = verInfo
serviceStateTableNode = descDoc.find("serviceStateTable",
namespaces=namespaces)
variablesTable, typesTable, eventsTable = self._locked_process_node_state_table(
serviceStateTableNode, namespaces=namespaces)
svcdesc["variablesTable"] = variablesTable
svcdesc["typesTable"] = typesTable
svcdesc["eventsTable"] = eventsTable
actionListNode = descDoc.find("actionList",
namespaces=namespaces)
actionsTable = self._locked_process_node_action_list(
actionListNode, namespaces=namespaces)
svcdesc["actionsTable"] = actionsTable
except:
print("Service Description Failure: %s" % sdurl)
raise
return svcdesc
def process_subscription_callback(self, sender_ip, sid: str, headers: dict,
body: str):
"""
This is used by the subscription callback thread to handoff work packets to the a worker thread
and completes the processing of the subscription callback and routing to a UpnpRootDevice and
UpnpServiceProxy instance.
:param sender_ip: The IP address of the entity sending the update
:param sid: The Subscription ID (sid) associated with the subscription callback
:param headers: The HTTP headers contained in the callback response.
:param body: The body content of the HTTP subscription callback.
"""
# pylint: disable=unused-argument
service = None
self._device_lock.acquire()
try:
if sid in self._sid_to_service_lookup:
service = self._sid_to_service_lookup[sid]
finally:
self._device_lock.release()
if service is not None:
docTree = ElementTree(xml_fromstring(body))
psetNode = docTree.getroot()
if psetNode is not None and psetNode.tag == "{%s}propertyset" % NS_UPNP_EVENT:
propertyNodeList = psetNode.findall("{%s}property" %
NS_UPNP_EVENT)
service._update_event_variables(sender_ip, self._usn_dev,
propertyNodeList) # pylint: disable=protected-access
return
def _get_metadata(self):
"""
Retrieves metadata about the DHCD DFD Quick Base app and its member tables
(including field metadata and relationships) and saves this in two CSV files.
Also, for each unique data id corresponding to a table, (1) builds a field
reference list of all relevant fields, and (2) sets the query parameter string
(including the sort field parameter) used when saving table data in get_data(...).
"""
output_path_dir = os.path.dirname(
self.output_paths[self._available_unique_data_ids[0]])
output_path_app_metadata = os.path.join(output_path_dir,
'_dhcd_dfd_app_metadata.csv')
output_path_table_metadata = os.path.join(
output_path_dir, '_dhcd_dfd_table_metadata.csv')
app_metadata_result = self.get('/' + self._app_dbid,
params=DhcdApiConn.PARAMS_METADATA)
app_tables_metadata_xml = xml_fromstring(
app_metadata_result.text).findall('./table/chdbids/chdbid')
app_metadata = OrderedDict()
table_metadata = OrderedDict()
field_count = 0
for app_table_metadata in app_tables_metadata_xml:
table_dbid = app_table_metadata.text
table_metadata_result = self.get(
'/' + table_dbid, params=DhcdApiConn.PARAMS_METADATA)
# Strip out singly-occurring line break tags to prevent truncation of multi-line formulas
table_metadata_full = table_metadata_result.text.replace(
"<BR/>\n<BR/>", "<br />\n<br />")
table_metadata_full = table_metadata_result.text.replace(
"<BR/>", "")
table_metadata_xml_root = xml_fromstring(table_metadata_full)
errcode = int(table_metadata_xml_root.find('./errcode').text)
if errcode == 0:
table_metadata_xml_orig = table_metadata_xml_root.find(
'./table/original')
table_name = table_metadata_xml_root.find('./table/name').text
table_name_snake_case = table_name.lower().translate(
self._identifier_translation_map)
unique_data_id = None
if 'dhcd_dfd_' + table_name_snake_case in self._available_unique_data_ids:
unique_data_id = 'dhcd_dfd_' + table_name_snake_case
self._fields[unique_data_id] = []
table_metadata_xml_fields = table_metadata_xml_root.findall(
'./table/fields/field')
table_metadata[table_dbid] = OrderedDict()
field_line_start = field_count + 2
for field_xml in table_metadata_xml_fields:
fid = int(field_xml.get('id'))
table_metadata[table_dbid][fid] = OrderedDict()
field_label = field_xml.find('label').text
field_name = field_label.lower().translate(
self._identifier_translation_map)
# For any fields that belong to composite fields (e.g. address component fields),
# resolve the full field name by prepending the parent field name
parent_fid = None
if field_xml.find('parentFieldID') is not None:
parent_fid = int(
field_xml.find('parentFieldID').text)
if parent_fid in table_metadata[table_dbid]:
parent_field_name = table_metadata[table_dbid][
parent_fid]['field_name']
else:
parent_field_label = table_metadata_xml_root.find(
"./table/fields/field[@id='{}']/label".
format(str(parent_fid))).text
parent_field_name = parent_field_label.lower(
).translate(self._identifier_translation_map)
if parent_field_name[0].isdigit():
parent_field_name = '_' + parent_field_name
field_name = '__'.join(
[parent_field_name, field_name])
if field_name[0].isdigit():
field_name = '_' + field_name
# For any composite fields (e.g. address fields), get child/component fields
child_fids = []
for child_field in field_xml.findall(
'./compositeFields/compositeField'):
child_fids.append(child_field.get('id'))
child_fids = '|'.join(
child_fids) if len(child_fids) > 0 else None
table_metadata[table_dbid][fid][
'table_name'] = table_name
table_metadata[table_dbid][fid][
'field_name'] = field_name
table_metadata[table_dbid][fid][
'field_label'] = field_label
table_metadata[table_dbid][fid]['field_id'] = str(fid)
table_metadata[table_dbid][fid][
'field_type'] = field_xml.get('field_type')
table_metadata[table_dbid][fid][
'base_type'] = field_xml.get('base_type')
table_metadata[table_dbid][fid][
'appears_by_default'] = field_xml.find(
'appears_by_default').text
table_metadata[table_dbid][fid][
'composite_field_parent_fid'] = parent_fid
table_metadata[table_dbid][fid][
'composite_field_child_fids'] = child_fids
table_metadata[table_dbid][fid][
'mode'] = field_xml.get('mode')
table_metadata[table_dbid][fid]['formula'] = None
if field_xml.find('formula') is not None:
table_metadata[table_dbid][fid][
'formula'] = field_xml.find('formula').text
table_metadata[table_dbid][fid]['choices'] = None
if field_xml.find('choices') is not None:
table_metadata[table_dbid][fid]['choices'] = ""
for choice in field_xml.findall(
'./choices/choice'):
table_metadata[table_dbid][fid]['choices'] += "\n" + choice.text \
if len(table_metadata[table_dbid][fid]['choices']) > 0 \
else choice.text
table_metadata[table_dbid][fid][
'lookup_target_fid'] = None
table_metadata[table_dbid][fid][
'lookup_source_fid'] = None
if table_metadata[table_dbid][fid]['mode'] == 'lookup':
if field_xml.find('lutfid') is not None:
table_metadata[table_dbid][fid][
'lookup_target_fid'] = field_xml.find(
'lutfid').text
if field_xml.find('lusfid') is not None:
table_metadata[table_dbid][fid][
'lookup_source_fid'] = field_xml.find(
'lusfid').text
table_metadata[table_dbid][fid][
'dblink_target_dbid'] = None
table_metadata[table_dbid][fid][
'dblink_target_fid'] = None
table_metadata[table_dbid][fid][
'dblink_source_fid'] = None
if table_metadata[table_dbid][fid]['mode'] == 'virtual' and \
table_metadata[table_dbid][fid]['field_type'] == 'dblink':
if field_xml.find('target_dbid') is not None:
table_metadata[table_dbid][fid][
'dblink_target_dbid'] = field_xml.find(
'target_dbid').text
if field_xml.find('target_fid') is not None:
table_metadata[table_dbid][fid][
'dblink_target_fid'] = field_xml.find(
'target_fid').text
if field_xml.find('source_fid') is not None:
table_metadata[table_dbid][fid][
'dblink_source_fid'] = field_xml.find(
'source_fid').text
table_metadata[table_dbid][fid][
'fkey_table_app_dbid'] = None
table_metadata[table_dbid][fid][
'fkey_table_alias'] = None
if field_xml.find('mastag') is not None:
fkey_ref = field_xml.find('mastag').text.split('.')
if len(fkey_ref) == 2:
table_metadata[table_dbid][fid][
'fkey_table_app_dbid'] = fkey_ref[0]
table_metadata[table_dbid][fid][
'fkey_table_alias'] = fkey_ref[1].lower()
else:
table_metadata[table_dbid][fid][
'fkey_table_app_dbid'] = None
table_metadata[table_dbid][fid][
'fkey_table_alias'] = fkey_ref[0].lower()
table_metadata[table_dbid][fid][
'field_help'] = field_xml.find('fieldhelp').text
# For each unique data id corresponding to a table,
# build a list of all relevant fields
if unique_data_id is not None and \
(INCLUDE_ALL_FIELDS[unique_data_id] or \
table_metadata[table_dbid][fid]['appears_by_default'] == '1'):
self._fields[unique_data_id].append(field_name)
field_count += 1
field_line_end = field_count + 1
app_metadata[table_dbid] = OrderedDict([
('table_name', table_name), ('table_dbid', table_dbid),
('table_alias', app_table_metadata.get('name')),
('key_fid',
table_metadata_xml_orig.find('key_fid').text),
('default_sort_fid',
table_metadata_xml_orig.find('def_sort_fid').text),
('default_sort_order',
table_metadata_xml_orig.find('def_sort_order').text),
('single_record_name',
table_metadata_xml_orig.find(
'single_record_name').text),
('plural_record_name',
table_metadata_xml_orig.find(
'plural_record_name').text),
('field_metadata_line_start', field_line_start),
('field_metadata_line_end', field_line_end)
])
if unique_data_id is not None and unique_data_id in self._fields:
# While not strictly a field, Quick Base always includes final 'update_id':
self._fields[unique_data_id].append('update_id')
# Set the query parameter string (including the sort field parameter):
if INCLUDE_ALL_FIELDS[unique_data_id]:
self._params[
unique_data_id] = DhcdApiConn.PARAMS_DATA_ALL_FIELDS
else:
self._params[
unique_data_id] = DhcdApiConn.PARAMS_DATA_DEFAULT_FIELDS
self._params[unique_data_id]['slist'] = app_metadata[
table_dbid]['default_sort_fid']
all_tables_field_metadata = [ list(field_metadata_row.values()) \
for all_field_metadata in table_metadata.values() \
for field_metadata_row in all_field_metadata.values() ]
self.result_to_csv(TABLE_METADATA_FIELDS, all_tables_field_metadata,
output_path_table_metadata)
self.result_to_csv(
APP_METADATA_FIELDS,
list(list(d.values()) for d in app_metadata.values()),
output_path_app_metadata)
def parse_response_error_for_upnp(self,
action_name,
content,
status_code,
extra=None,
encoding=None,
envelope_attrib=None,
typed=None):
"""
Parse response error for a upnp response.
"""
register_namespace('', None)
if encoding is None:
encoding = self._encoding
if envelope_attrib is None:
envelope_attrib = self._envelope_attrib
if typed is None:
typed = self._typed
try:
docNode = xml_fromstring(content)
except ParseError:
# Try removing any extra XML declarations in case there are more than one.
# This sometimes happens when a device sends its own XML config files.
content = remove_extraneous_xml_declarations(content)
docNode = xml_fromstring(content)
except ValueError:
# This can occur when requests returns a `str` (unicode) but there's also an XML
# declaration, which lxml doesn't like.
docNode = xml_fromstring(content.encode('utf8'))
resp_body = None
if typed:
resp_body = docNode.find(".//{%s}Fault" % (NS_SOAP_ENV, ))
else:
resp_body = docNode.find(".//Fault")
if resp_body is None:
msg = (
'Returned XML did not include an element which matches namespace %r and tag name'
' \'%sFault\'.' % (typed, action_name))
print(msg + '\n' + xml_tostring(
docNode, short_empty_elements=False).decode('utf8'))
raise SOAPProtocolError(msg)
# Lets try to extract the XML response error information
try:
faultCode = resp_body.find(".//faultcode").text
faultString = resp_body.find(".//faultstring").text
detail = resp_body.find(".//detail")
upnpErrorNode = detail.find(".//{%s}UPnPError" % NS_UPNP_CONTROL)
errorCode = int(
upnpErrorNode.find(".//{%s}errorCode" % NS_UPNP_CONTROL).text)
errorDescription = upnpErrorNode.find(".//{%s}errorDescription" %
NS_UPNP_CONTROL)
if errorDescription is None:
errorDescription = UPNP_ERROR_TEST_LOOKUP.get(
errorCode, "Unknown error.")
except Exception as xcpt:
errmsg = "Unable to process xml response: status=%r\n%s" % (
status_code, content)
if extra is not None:
errmsg += "EXTRA:\n%s" % extra
raise SOAPProtocolError(errmsg) from xcpt
return errorCode, errorDescription
def dir(self, cp_code=None, path=None, params=None):
params = params or {}
params['action'] = Actions.DIR
# Making the request
response, status = self.send(cp_code, path, params)
# Transform the element into a well know dict format
# It will look for these attributes:
# 'type', 'name', 'mtime' and, if present, also include the
# 'size', 'target' attributes
def dir_action_entry(element):
attribs = dict(**element.attrib)
entry = {
'type':
attribs['type'],
'name':
attribs['name'],
'mtime':
attribs['mtime'],
'path':
path_join(path, attribs['name']) if path else attribs['name']
}
# Optional keys
for k in (
'size',
'target',
):
try:
entry[k] = attribs[k]
except KeyError:
pass
# Integer keys
for k in (
'size',
'mtime',
):
try:
entry[k] = int(entry[k])
except (
KeyError,
ValueError,
):
pass
# Datetime keys
for k in ('mtime', ):
try:
entry[k] = datetime.fromtimestamp(entry[k])
except (
KeyError,
ValueError,
):
pass
return entry
if status == 200:
try:
tree = xml_fromstring(response)
return [
dir_action_entry(element)
for element in tree.findall('file')
]
except ParseError, parse_error:
raise AkamaiResponseMalformedException(str(parse_error))
def generate_service_proxies(svc_desc_directory: str,
svc_proxy_directory: str):
"""
Processes the XML service description documents in the description documents folder and generates the
service proxy modules. Then outputs the generated proxy modules to the service proxy foloder specified.
:param svc_desc_directory: The directory that contains the service description documents to process.
:param svc_proxy_directory: The directory that is the output directory for the service proxy modules.
"""
for dirpath, _, filenames in os.walk(svc_desc_directory, topdown=True):
for nxtfile in filenames:
serviceType, nxtfile_ext = os.path.splitext(nxtfile)
if nxtfile_ext != ".xml":
continue
serviceManufacturer = os.path.basename(dirpath)
svc_content = None
fullpath = os.path.join(dirpath, nxtfile)
with open(fullpath, 'r') as xf:
svc_content = xf.read()
docNode = xml_fromstring(svc_content)
if docNode is not None:
namespaces = None
doc_node_tag = docNode.tag
if doc_node_tag.find("}") > 0:
default_ns = doc_node_tag[doc_node_tag.find("{") +
1:doc_node_tag.find("}")]
namespaces = {"": default_ns}
variablesTable = {}
typesTable = {}
eventsTable = {}
svcStateTableNode = docNode.find("serviceStateTable",
namespaces=namespaces)
if svcStateTableNode is not None:
variablesTable, typesTable, eventsTable = process_service_state_table(
svcStateTableNode, namespaces=namespaces)
if serviceType.find("DeviceProperties") > 0:
print("found")
actionsTable = {}
actionListNode = docNode.find("actionList",
namespaces=namespaces)
if actionListNode is not None:
actionsTable = process_action_list(actionListNode,
namespaces=namespaces)
generate_upnp_service_proxy(svc_proxy_directory,
serviceManufacturer, serviceType,
variablesTable, typesTable,
eventsTable, actionsTable)
else:
errmsg = "WARNING: No serice node found in file:\n %s\n" % fullpath
print(errmsg, file=sys.stderr)
return
