def parse_response(self, action_name, content, encoding=None, envelope_attrib=None, typed=None): """ Parses a response from the server with the given action name and content. """ register_namespace('', None) if encoding is None: encoding = self._encoding if envelope_attrib is None: envelope_attrib = self._envelope_attrib if typed is None: typed = self._typed try: docNode = xml_fromstring(content) except ParseError: # Try removing any extra XML declarations in case there are more than one. # This sometimes happens when a device sends its own XML config files. content = remove_extraneous_xml_declarations(content) docNode = xml_fromstring(content) except ValueError: # This can occur when requests returns a `str` (unicode) but there's also an XML # declaration, which lxml doesn't like. docNode = xml_fromstring(content.encode('utf8')) resp_body = None if typed: resp_body = docNode.find(".//{%s}%sResponse" % (typed, action_name)) else: resp_body = docNode.find(".//%sResponse" % action_name) if resp_body is None: msg = ( 'Returned XML did not include an element which matches namespace %r and tag name' ' \'%sResponse\'.' % (typed, action_name)) print(msg + '\n' + xml_tostring( docNode, short_empty_elements=False).decode('utf8')) raise SOAPProtocolError(msg) # Sometimes devices return XML strings as their argument values without escaping them with # CDATA. This checks to see if the argument has been parsed as XML and un-parses it if so. resp_dict = {} for arg in resp_body.getchildren(): children = arg.getchildren() if children: resp_dict[arg.tag] = "\n".join( xml_tostring(x) for x in children) else: if arg.text is None: resp_dict[arg.tag] = "" else: resp_dict[arg.tag] = arg.text return resp_dict
def stat(self, cp_code=None, path=''): params = {} params['action'] = Actions.STAT response, status = self.send(cp_code, path, params) if status == 200: try: stats = xml_fromstring(response) element = stats.find('file') return self.__dir_action_entry(element, path) except ExpatError, parse_error: raise AkamaiResponseMalformedException(str(parse_error))
def du(self, cp_code=None, path=None, params=None): params = params or {} params['action'] = Actions.DU # Making the request response, status = self.send(cp_code, path, params) if status == 200: try: tree = xml_fromstring(response) info = tree.find('du-info').attrib return {'files': int(info['files']), 'bytes': int(info['bytes'])} except ParseError, parse_error: raise AkamaiResponseMalformedException(str(parse_error))
def get_data(self, unique_data_ids=None, sample=False, output_type='csv', **kwargs): """ Returns a JSON object of the entire data set. """ data_json = None db = kwargs.get('db', None) if unique_data_ids is None: unique_data_ids = self._available_unique_data_ids for u in unique_data_ids: if (u not in self._available_unique_data_ids): logger.info( " The unique_data_id '{}' is not supported by the DhcdApiConn" .format(u)) else: result = self.get(self._urls[u], params=self._params[u]) if result.status_code != 200: err = "An error occurred during request: status {0}" logger.exception(err.format(result.status_code)) continue data_xml_root = xml_fromstring(result.text) data_xml_records = data_xml_root.findall('record') data_json = xml_to_json.data(data_xml_root) results = [ DhcdResult({e.tag: e.text for e in list(r)}, self._fields[u]).data for r in data_xml_records ] self.result_to_csv(self._fields[u], results, self.output_paths[u]) #Convert to format expected by database if u == 'dhcd_dfd_properties': self.create_project_subsidy_csv('dhcd_dfd_properties', PROJECT_FIELDS_MAP, SUBSIDY_FIELDS_MAP, PROJECT_ADDRE_FIELDS_MAP, db)
def dir(self, cp_code=None, path=None, params=None): params = params or {} params['action'] = Actions.DIR # Making the request response, status = self.send(cp_code, path, params) # Transform the element into a well know dict format # It will look for these attributes: # 'type', 'name', 'mtime' and, if present, also include the # 'size', 'target' attributes def dir_action_entry(element): attribs = dict(**element.attrib) entry = { 'type': attribs['type'], 'name': attribs['name'], 'mtime': attribs['mtime'], 'path': path_join(path, attribs['name']) if path else attribs['name'] } # Optional keys for k in ('size', 'target', ): try: entry[k] = attribs[k] except KeyError: pass # Integer keys for k in ('size', 'mtime', ): try: entry[k] = int(entry[k]) except (KeyError, ValueError, ): pass # Datetime keys for k in ('mtime', ): try: entry[k] = datetime.fromtimestamp(entry[k]) except (KeyError, ValueError, ): pass return entry if status == 200: try: tree = xml_fromstring(response) return [dir_action_entry(element) for element in tree.findall('file')] except ParseError, parse_error: raise AkamaiResponseMalformedException(str(parse_error))
def dir(self, cp_code=None, path=None, params=None): params = params or {} params['action'] = Actions.DIR # Making the request response, status = self.send(cp_code, path, params) if status == 200: try: tree = xml_fromstring(response) return [ self.__dir_action_entry(element, path) for element in tree.findall('file') ] except ExpatError, parse_error: raise AkamaiResponseMalformedException(str(parse_error))
def du(self, cp_code=None, path=None, params=None): params = params or {} params['action'] = Actions.DU # Making the request response, status = self.send(cp_code, path, params) if status == 200: try: tree = xml_fromstring(response) info = tree.find('du-info').attrib return { 'files': int(info['files']), 'bytes': int(info['bytes']) } except ParseError, parse_error: raise AkamaiResponseMalformedException(str(parse_error))
def device_description_load(location: str) -> Union[ElementTree, None]: """ Gets the description document from the specified URL and loads the contents into an XML ElementTree. :param location: The url of where to get the device description. :returns: The XML ElementTree for the XML content from the device description or None. """ docTree = None resp = requests.get(location) if resp.status_code == 200: xmlcontent = resp.content docTree = ElementTree(xml_fromstring(xmlcontent)) return docTree
def _locked_process_full_service_description(self, sdurl: str): """ Downloads and processes the service description document. :param sdurl: The url of the service description document to process. """ svcdesc = None resp = requests.get(sdurl) if resp.status_code == 200: svcdesc = {} namespaces = {"": UPNP_SERVICE1_NAMESPACE} try: xml_content = resp.text descDoc = xml_fromstring(xml_content) specVersionNode = descDoc.find("specVersion", namespaces=namespaces) verInfo = self._locked_process_node_spec_version( specVersionNode, namespaces=namespaces) svcdesc["specVersion"] = verInfo serviceStateTableNode = descDoc.find("serviceStateTable", namespaces=namespaces) variablesTable, typesTable, eventsTable = self._locked_process_node_state_table( serviceStateTableNode, namespaces=namespaces) svcdesc["variablesTable"] = variablesTable svcdesc["typesTable"] = typesTable svcdesc["eventsTable"] = eventsTable actionListNode = descDoc.find("actionList", namespaces=namespaces) actionsTable = self._locked_process_node_action_list( actionListNode, namespaces=namespaces) svcdesc["actionsTable"] = actionsTable except: print("Service Description Failure: %s" % sdurl) raise return svcdesc
def process_subscription_callback(self, sender_ip, sid: str, headers: dict, body: str): """ This is used by the subscription callback thread to handoff work packets to the a worker thread and completes the processing of the subscription callback and routing to a UpnpRootDevice and UpnpServiceProxy instance. :param sender_ip: The IP address of the entity sending the update :param sid: The Subscription ID (sid) associated with the subscription callback :param headers: The HTTP headers contained in the callback response. :param body: The body content of the HTTP subscription callback. """ # pylint: disable=unused-argument service = None self._device_lock.acquire() try: if sid in self._sid_to_service_lookup: service = self._sid_to_service_lookup[sid] finally: self._device_lock.release() if service is not None: docTree = ElementTree(xml_fromstring(body)) psetNode = docTree.getroot() if psetNode is not None and psetNode.tag == "{%s}propertyset" % NS_UPNP_EVENT: propertyNodeList = psetNode.findall("{%s}property" % NS_UPNP_EVENT) service._update_event_variables(sender_ip, self._usn_dev, propertyNodeList) # pylint: disable=protected-access return
def _get_metadata(self): """ Retrieves metadata about the DHCD DFD Quick Base app and its member tables (including field metadata and relationships) and saves this in two CSV files. Also, for each unique data id corresponding to a table, (1) builds a field reference list of all relevant fields, and (2) sets the query parameter string (including the sort field parameter) used when saving table data in get_data(...). """ output_path_dir = os.path.dirname( self.output_paths[self._available_unique_data_ids[0]]) output_path_app_metadata = os.path.join(output_path_dir, '_dhcd_dfd_app_metadata.csv') output_path_table_metadata = os.path.join( output_path_dir, '_dhcd_dfd_table_metadata.csv') app_metadata_result = self.get('/' + self._app_dbid, params=DhcdApiConn.PARAMS_METADATA) app_tables_metadata_xml = xml_fromstring( app_metadata_result.text).findall('./table/chdbids/chdbid') app_metadata = OrderedDict() table_metadata = OrderedDict() field_count = 0 for app_table_metadata in app_tables_metadata_xml: table_dbid = app_table_metadata.text table_metadata_result = self.get( '/' + table_dbid, params=DhcdApiConn.PARAMS_METADATA) # Strip out singly-occurring line break tags to prevent truncation of multi-line formulas table_metadata_full = table_metadata_result.text.replace( "<BR/>\n<BR/>", "<br />\n<br />") table_metadata_full = table_metadata_result.text.replace( "<BR/>", "") table_metadata_xml_root = xml_fromstring(table_metadata_full) errcode = int(table_metadata_xml_root.find('./errcode').text) if errcode == 0: table_metadata_xml_orig = table_metadata_xml_root.find( './table/original') table_name = table_metadata_xml_root.find('./table/name').text table_name_snake_case = table_name.lower().translate( self._identifier_translation_map) unique_data_id = None if 'dhcd_dfd_' + table_name_snake_case in self._available_unique_data_ids: unique_data_id = 'dhcd_dfd_' + table_name_snake_case self._fields[unique_data_id] = [] table_metadata_xml_fields = table_metadata_xml_root.findall( './table/fields/field') table_metadata[table_dbid] = OrderedDict() field_line_start = field_count + 2 for field_xml in table_metadata_xml_fields: fid = int(field_xml.get('id')) table_metadata[table_dbid][fid] = OrderedDict() field_label = field_xml.find('label').text field_name = field_label.lower().translate( self._identifier_translation_map) # For any fields that belong to composite fields (e.g. address component fields), # resolve the full field name by prepending the parent field name parent_fid = None if field_xml.find('parentFieldID') is not None: parent_fid = int( field_xml.find('parentFieldID').text) if parent_fid in table_metadata[table_dbid]: parent_field_name = table_metadata[table_dbid][ parent_fid]['field_name'] else: parent_field_label = table_metadata_xml_root.find( "./table/fields/field[@id='{}']/label". format(str(parent_fid))).text parent_field_name = parent_field_label.lower( ).translate(self._identifier_translation_map) if parent_field_name[0].isdigit(): parent_field_name = '_' + parent_field_name field_name = '__'.join( [parent_field_name, field_name]) if field_name[0].isdigit(): field_name = '_' + field_name # For any composite fields (e.g. address fields), get child/component fields child_fids = [] for child_field in field_xml.findall( './compositeFields/compositeField'): child_fids.append(child_field.get('id')) child_fids = '|'.join( child_fids) if len(child_fids) > 0 else None table_metadata[table_dbid][fid][ 'table_name'] = table_name table_metadata[table_dbid][fid][ 'field_name'] = field_name table_metadata[table_dbid][fid][ 'field_label'] = field_label table_metadata[table_dbid][fid]['field_id'] = str(fid) table_metadata[table_dbid][fid][ 'field_type'] = field_xml.get('field_type') table_metadata[table_dbid][fid][ 'base_type'] = field_xml.get('base_type') table_metadata[table_dbid][fid][ 'appears_by_default'] = field_xml.find( 'appears_by_default').text table_metadata[table_dbid][fid][ 'composite_field_parent_fid'] = parent_fid table_metadata[table_dbid][fid][ 'composite_field_child_fids'] = child_fids table_metadata[table_dbid][fid][ 'mode'] = field_xml.get('mode') table_metadata[table_dbid][fid]['formula'] = None if field_xml.find('formula') is not None: table_metadata[table_dbid][fid][ 'formula'] = field_xml.find('formula').text table_metadata[table_dbid][fid]['choices'] = None if field_xml.find('choices') is not None: table_metadata[table_dbid][fid]['choices'] = "" for choice in field_xml.findall( './choices/choice'): table_metadata[table_dbid][fid]['choices'] += "\n" + choice.text \ if len(table_metadata[table_dbid][fid]['choices']) > 0 \ else choice.text table_metadata[table_dbid][fid][ 'lookup_target_fid'] = None table_metadata[table_dbid][fid][ 'lookup_source_fid'] = None if table_metadata[table_dbid][fid]['mode'] == 'lookup': if field_xml.find('lutfid') is not None: table_metadata[table_dbid][fid][ 'lookup_target_fid'] = field_xml.find( 'lutfid').text if field_xml.find('lusfid') is not None: table_metadata[table_dbid][fid][ 'lookup_source_fid'] = field_xml.find( 'lusfid').text table_metadata[table_dbid][fid][ 'dblink_target_dbid'] = None table_metadata[table_dbid][fid][ 'dblink_target_fid'] = None table_metadata[table_dbid][fid][ 'dblink_source_fid'] = None if table_metadata[table_dbid][fid]['mode'] == 'virtual' and \ table_metadata[table_dbid][fid]['field_type'] == 'dblink': if field_xml.find('target_dbid') is not None: table_metadata[table_dbid][fid][ 'dblink_target_dbid'] = field_xml.find( 'target_dbid').text if field_xml.find('target_fid') is not None: table_metadata[table_dbid][fid][ 'dblink_target_fid'] = field_xml.find( 'target_fid').text if field_xml.find('source_fid') is not None: table_metadata[table_dbid][fid][ 'dblink_source_fid'] = field_xml.find( 'source_fid').text table_metadata[table_dbid][fid][ 'fkey_table_app_dbid'] = None table_metadata[table_dbid][fid][ 'fkey_table_alias'] = None if field_xml.find('mastag') is not None: fkey_ref = field_xml.find('mastag').text.split('.') if len(fkey_ref) == 2: table_metadata[table_dbid][fid][ 'fkey_table_app_dbid'] = fkey_ref[0] table_metadata[table_dbid][fid][ 'fkey_table_alias'] = fkey_ref[1].lower() else: table_metadata[table_dbid][fid][ 'fkey_table_app_dbid'] = None table_metadata[table_dbid][fid][ 'fkey_table_alias'] = fkey_ref[0].lower() table_metadata[table_dbid][fid][ 'field_help'] = field_xml.find('fieldhelp').text # For each unique data id corresponding to a table, # build a list of all relevant fields if unique_data_id is not None and \ (INCLUDE_ALL_FIELDS[unique_data_id] or \ table_metadata[table_dbid][fid]['appears_by_default'] == '1'): self._fields[unique_data_id].append(field_name) field_count += 1 field_line_end = field_count + 1 app_metadata[table_dbid] = OrderedDict([ ('table_name', table_name), ('table_dbid', table_dbid), ('table_alias', app_table_metadata.get('name')), ('key_fid', table_metadata_xml_orig.find('key_fid').text), ('default_sort_fid', table_metadata_xml_orig.find('def_sort_fid').text), ('default_sort_order', table_metadata_xml_orig.find('def_sort_order').text), ('single_record_name', table_metadata_xml_orig.find( 'single_record_name').text), ('plural_record_name', table_metadata_xml_orig.find( 'plural_record_name').text), ('field_metadata_line_start', field_line_start), ('field_metadata_line_end', field_line_end) ]) if unique_data_id is not None and unique_data_id in self._fields: # While not strictly a field, Quick Base always includes final 'update_id': self._fields[unique_data_id].append('update_id') # Set the query parameter string (including the sort field parameter): if INCLUDE_ALL_FIELDS[unique_data_id]: self._params[ unique_data_id] = DhcdApiConn.PARAMS_DATA_ALL_FIELDS else: self._params[ unique_data_id] = DhcdApiConn.PARAMS_DATA_DEFAULT_FIELDS self._params[unique_data_id]['slist'] = app_metadata[ table_dbid]['default_sort_fid'] all_tables_field_metadata = [ list(field_metadata_row.values()) \ for all_field_metadata in table_metadata.values() \ for field_metadata_row in all_field_metadata.values() ] self.result_to_csv(TABLE_METADATA_FIELDS, all_tables_field_metadata, output_path_table_metadata) self.result_to_csv( APP_METADATA_FIELDS, list(list(d.values()) for d in app_metadata.values()), output_path_app_metadata)
def parse_response_error_for_upnp(self, action_name, content, status_code, extra=None, encoding=None, envelope_attrib=None, typed=None): """ Parse response error for a upnp response. """ register_namespace('', None) if encoding is None: encoding = self._encoding if envelope_attrib is None: envelope_attrib = self._envelope_attrib if typed is None: typed = self._typed try: docNode = xml_fromstring(content) except ParseError: # Try removing any extra XML declarations in case there are more than one. # This sometimes happens when a device sends its own XML config files. content = remove_extraneous_xml_declarations(content) docNode = xml_fromstring(content) except ValueError: # This can occur when requests returns a `str` (unicode) but there's also an XML # declaration, which lxml doesn't like. docNode = xml_fromstring(content.encode('utf8')) resp_body = None if typed: resp_body = docNode.find(".//{%s}Fault" % (NS_SOAP_ENV, )) else: resp_body = docNode.find(".//Fault") if resp_body is None: msg = ( 'Returned XML did not include an element which matches namespace %r and tag name' ' \'%sFault\'.' % (typed, action_name)) print(msg + '\n' + xml_tostring( docNode, short_empty_elements=False).decode('utf8')) raise SOAPProtocolError(msg) # Lets try to extract the XML response error information try: faultCode = resp_body.find(".//faultcode").text faultString = resp_body.find(".//faultstring").text detail = resp_body.find(".//detail") upnpErrorNode = detail.find(".//{%s}UPnPError" % NS_UPNP_CONTROL) errorCode = int( upnpErrorNode.find(".//{%s}errorCode" % NS_UPNP_CONTROL).text) errorDescription = upnpErrorNode.find(".//{%s}errorDescription" % NS_UPNP_CONTROL) if errorDescription is None: errorDescription = UPNP_ERROR_TEST_LOOKUP.get( errorCode, "Unknown error.") except Exception as xcpt: errmsg = "Unable to process xml response: status=%r\n%s" % ( status_code, content) if extra is not None: errmsg += "EXTRA:\n%s" % extra raise SOAPProtocolError(errmsg) from xcpt return errorCode, errorDescription
def dir(self, cp_code=None, path=None, params=None): params = params or {} params['action'] = Actions.DIR # Making the request response, status = self.send(cp_code, path, params) # Transform the element into a well know dict format # It will look for these attributes: # 'type', 'name', 'mtime' and, if present, also include the # 'size', 'target' attributes def dir_action_entry(element): attribs = dict(**element.attrib) entry = { 'type': attribs['type'], 'name': attribs['name'], 'mtime': attribs['mtime'], 'path': path_join(path, attribs['name']) if path else attribs['name'] } # Optional keys for k in ( 'size', 'target', ): try: entry[k] = attribs[k] except KeyError: pass # Integer keys for k in ( 'size', 'mtime', ): try: entry[k] = int(entry[k]) except ( KeyError, ValueError, ): pass # Datetime keys for k in ('mtime', ): try: entry[k] = datetime.fromtimestamp(entry[k]) except ( KeyError, ValueError, ): pass return entry if status == 200: try: tree = xml_fromstring(response) return [ dir_action_entry(element) for element in tree.findall('file') ] except ParseError, parse_error: raise AkamaiResponseMalformedException(str(parse_error))
def generate_service_proxies(svc_desc_directory: str, svc_proxy_directory: str): """ Processes the XML service description documents in the description documents folder and generates the service proxy modules. Then outputs the generated proxy modules to the service proxy foloder specified. :param svc_desc_directory: The directory that contains the service description documents to process. :param svc_proxy_directory: The directory that is the output directory for the service proxy modules. """ for dirpath, _, filenames in os.walk(svc_desc_directory, topdown=True): for nxtfile in filenames: serviceType, nxtfile_ext = os.path.splitext(nxtfile) if nxtfile_ext != ".xml": continue serviceManufacturer = os.path.basename(dirpath) svc_content = None fullpath = os.path.join(dirpath, nxtfile) with open(fullpath, 'r') as xf: svc_content = xf.read() docNode = xml_fromstring(svc_content) if docNode is not None: namespaces = None doc_node_tag = docNode.tag if doc_node_tag.find("}") > 0: default_ns = doc_node_tag[doc_node_tag.find("{") + 1:doc_node_tag.find("}")] namespaces = {"": default_ns} variablesTable = {} typesTable = {} eventsTable = {} svcStateTableNode = docNode.find("serviceStateTable", namespaces=namespaces) if svcStateTableNode is not None: variablesTable, typesTable, eventsTable = process_service_state_table( svcStateTableNode, namespaces=namespaces) if serviceType.find("DeviceProperties") > 0: print("found") actionsTable = {} actionListNode = docNode.find("actionList", namespaces=namespaces) if actionListNode is not None: actionsTable = process_action_list(actionListNode, namespaces=namespaces) generate_upnp_service_proxy(svc_proxy_directory, serviceManufacturer, serviceType, variablesTable, typesTable, eventsTable, actionsTable) else: errmsg = "WARNING: No serice node found in file:\n %s\n" % fullpath print(errmsg, file=sys.stderr) return