def getstd(self): c=self.elem.find('LeedSummary').findall('Eap245PerformanceRatingMethodCompliance') s=pd.DataFrame(pr.data(c)['Eap245PerformanceRatingMethodCompliance'])[:-1].melt(id_vars='name') s = s[s['value']>0] s = s.set_index(s.name+':'+s.variable)['value'] c = self.elem.find('SystemSummary').findall('TimeSetpointNotMet')[-1] s=s.append(pd.Series(pr.data(c))[1:]) s['EUI']=float(self.elem.find('AnnualBuildingUtilityPerformanceSummary').findall('SiteAndSourceEnergy')[1].find('EnergyPerTotalBuildingArea').text) coils=pr.data(self.elem.find('EquipmentSummary').findall('CoolingCoils'))['CoolingCoils'] s['ClCoilSens']=pd.DataFrame(coils,index=range(max(1,isinstance(coils,list)*len(coils))))[['NominalSensibleCapacity']].sum()[0] coils=pr.data(self.elem.find('EquipmentSummary').findall('CoolingCoils'))['CoolingCoils'] s['ClcoilLat']=pd.DataFrame(coils,index=range(max(1,isinstance(coils,list)*len(coils))))[['NominalLatentCapacity']].sum()[0] coils=pr.data(self.elem.find('EquipmentSummary').findall('HeatingCoils'))['HeatingCoils'] s['HtCoil']=pd.DataFrame(coils,index=range(max(1,isinstance(coils,list)*len(coils))))[['NominalTotalCapacity']].sum()[0] s['clgCFM'] = pd.DataFrame(pr.data(self.elem.find('HvacSizingSummary').findall('ZoneSensibleCooling'))['ZoneSensibleCooling'])['UserDesignAirFlow'].sum() s['oaCFM'] = pd.DataFrame(pr.data(self.elem.find('HvacSizingSummary').findall('ZoneSensibleCooling'))['ZoneSensibleCooling'])['MinimumOutdoorAirFlowRate'].sum() s['htgCFM'] = pd.DataFrame(pr.data(self.elem.find('HvacSizingSummary').findall('ZoneSensibleHeating'))['ZoneSensibleHeating'])['UserDesignAirFlow'].sum() return s
def _get_acl_rules(self, rest_device, address_type, acl_type, acl_name, sequences): """ Return list of rules configured for acl_name """ rules_list = [] method = address_type + '_access_list_' + acl_type + '_seq_get' config = (method, {acl_type: acl_name, 'resource_depth': 2}) output = rest_device._callback(config, handler='get_config') util = Util(output.data) for rcvd_seq in util.root.findall(".//seq"): if rcvd_seq is not None: seq_id = int(rcvd_seq.find('seq-id').text) if seq_id in sequences: sequences.remove(seq_id) pd = parker.data(rcvd_seq) new_pd = {} # Replace "-" with "_" for k, v in pd.iteritems(): nk = k.replace("-", "_") new_pd[nk] = v rules_list.append(new_pd) return rules_list
def download_subjects(self, job): # download subjects info('downloading subjects definition from "' + self.xml_stats_url + '"...') # download xml stats_xml = os.path.join(job.job_dir, 'subjects.xml') urllib.request.urlretrieve(self.xml_stats_url, stats_xml) # parse xml to json root = ET.parse(stats_xml).getroot() subjects = parker.data( root.find("./achievements"), preserve_root=True).get('achievements').get('achievement') store_json(subjects, job.subjects_file) # download icons info('downloading subjects...') i = 0 total = len(subjects) progress(i, total) for subject in subjects: subject_url = subject.get('iconClosed') urllib.request.urlretrieve( subject_url, os.path.join(job.subjects_dir, ntpath.basename(subject_url))) i = i + 1 progress(i, total) success('stored ' + str(len(glob.glob(os.path.join(job.subjects_dir, "*.jpg")))) + ' subjects to ' + job.subjects_dir)
def save_to_mongo(content): client = MongoClient() ## client = MongoClient("192.168.1.194", 27107) dbtemp = client.temp ## timestmp = datetime.datetime.now().strftime("%Y-%d-%d %H:%M:%S") timestmp = datetime.datetime.utcnow() xmlread = content.decode('ascii') xmlread = fromstring(xmlread) newfile = dumps(parker.data(xmlread)) data = loads(newfile) result = dbtemp.new_Ireq_data.insert_one(data) db = client.Ireqdata copycursor = db.req_data.find() ## db.req_archive.drop_indexes() ## db.req_archive.reindex() db.req_data.drop_indexes() db.req_data.reindex() for document in copycursor: db.req_archive.insert(document) print(document) cursor = dbtemp.new_Ireq_data.distinct( "Unit.Packet.Payload.ResultSet.Jobs.Job") for document in cursor: newreq = {"dateAdded": timestmp, "req": document} db.Ireq_data.insert_one(newreq) dbtemp.new_Ireq_data.drop()
def recv_data_object(self): frame = self.recv_frame() data_frame = self.frame_data_parse(frame) xml_data = self.frame_data_xml(data_frame) ret_val = dumps(p.data(fromstring(xml_data))) return ret_val
def annotation_xml_to_json(xml_file): f = open(xml_file, "rb") json_dict = keep_keys(parker.data(fromstring(f.read()))) if not "object" in json_dict: json_dict = {"object": {}} json_output = json.dumps(json_dict["object"]) f.close() return json_output
def _get_data(self): """Get the status from the phone system.""" raw_res = requests.get( STATUS_URL.format(self.hostname, self.username, self.password)) if not raw_res.status_code == 200: raise PyCiscoSPAError("Login Error: Bad HTTP status code.") self._data = parker.data(fromstring(raw_res.content))
def convert_response_to_json(self, response_xml): return { "data": parker.data(response_xml), "meta": { "version": version, "author": author, "id": generate_file_id() } }
def xml_to_json(path): with open(path, 'r') as r: root = fromstring(r.read()) root_attributes = {key.lower(): value.lower() for key, value in root.attrib.items()} if 'convention' in root_attributes: convention = root_attributes['convention'] if convention == 'parker': return parker.data(root) return badgerfish.data(root)
def transform_xml_to_json(xml_file_input, json_file_output): """Transform Single File""" with open(xml_file_input, 'r') as f1: xml_text = f1.read() # Create directory to store output file , if already exists do nothing os.makedirs(os.path.dirname(json_file_output), exist_ok=True) with open(json_file_output, 'w') as f2: json_text = dumps(parker.data(fromstring((xml_text)))) f2.write(json_text)
def get_stat(key): url = get_setting('fse_datafeed_stat') url = url.format(key, key) response = requests.get(url) if "Currently Closed for Maintenance" in response.text: raise RuntimeError( 'FSEconomy is currently down for maintenance. Please try again later.' ) clean = re.sub(r'<StatisticItems[^>]+>', '<StatisticItems>', response.text) data = parker.data(fromstring(clean)) return data['Statistic']
def parse_and_save_data(experiement_file): experiment_data = ExperimentData() experiment_data.experiment_file = experiement_file if "xml" in experiment_data.experiment_file.get_file_mime(): root = parse( StringIO(experiment_data.experiment_file.get_file_content())) experiment_data.data = dumps(parker.data(root.getroot())) experiment_data.parsing_method = 'XML' experiment_data.parsing_datetime = datetime.datetime.now() experiment_data.save()
def system_multicall(self, commands): body = "<?xml version='1.0'?><methodCall><methodName>system.multicall</methodName><params><param><value><array><data>" if commands: for c in commands: body += '<value>' + self.get_struct(c) + '</value>' body += "</data></array></value></param></params></methodCall>" scgi = Scgi(self.host_port) resp = scgi.post(body) resp = resp[resp.find('<'):] # strip headers # print(resp) data = parker.data(fromstring(resp), preserve_root=True) # convert to json return data
def test_empty(): # arrange file = open("./test-data/test_input/empty.xml", "r") file_text = file.read() expected_message = open("./test-data/expected_messages/empty.log", "r").read() # act & assert with pytest.raises(ParseError, match=r".*" + expected_message + ".*"): json_str = dumps(parker.data(fromstring(file_text)), preserve_root=True)
def convertPoiXML(ui, filepath): if validatePoiXML(filepath): poiTree = ET.parse(filepath) poiRoot = poiTree.getroot() poiDict = json.loads(json.dumps(pk.data(poiRoot))) return poiDict else: print('invalid POI XML (does not conform to POI schema)') QMessageBox.question( ui, "Error: Invalid File", "Provided file must be an XML that conforms to poiConfig,xsd (schema)", QMessageBox.Ok) return 0
def callZillowApi(): address = request.get_json() #zillow call payload = { "zws-id": config.zillow_key, "address": address["line1"], "citystatezip": address["locality"] + "," + address["countrySubd"] } resp = requests.get(zillow + "GetDeepSearchResults.htm", params=payload) json_data = json.loads(json.dumps(xmlparse.data(fromstring(resp.text)))) if json_data["message"]["code"] != 0: return "", 503 return json.dumps(json_data["response"]["results"]), 200
def getall(self): r = pr.data(self.elem) for ea in['BuildingName','EnvironmentName','WeatherFileLocationTitle' ,'ProgramVersion','SimulationTimestamp']: r.pop(ea,None) for ea in r: if not 'monthly' in ea: for j in ['for','note','footnote','General']: r[ea].pop(j,None) for j in r[ea]: if type(r[ea][j]) is list: self.__setattr__(j,pd.DataFrame(r[ea][j]).set_index('name')) else: self.__setattr__(j,pd.Series(r[ea][j]))
def test_invalid_xml(): # arrange input = open("./test-data/test_input/invalid.xml", "r") input_text = input.read() print("input_text: " + input_text) expected_message = open("./test-data/expected_messages/invalid_xml.log", "r").read() # act & assert with pytest.raises(ParseError, match=r".*" + expected_message + ".*"): json_str = dumps( parker.data(fromstring(input_text), preserve_root=False))
def call(self, method, params=None): body = "<?xml version='1.0'?><methodCall><methodName>" + \ method + "</methodName><params>" if params: for p in params: body += '<param><value><' + p[0] + '>' + str(p[1]) + '</' + p[0] + '></value></param>' body += "</params></methodCall>" scgi = Scgi(self.host_port) resp = scgi.post(body) resp = resp[resp.find('<'):] # strip headers # print(resp) data = parker.data(fromstring(resp), preserve_root=True) # convert to json return data
def __iter_extended_rows(self): from xml.etree.ElementTree import parse from xmljson import parker parsed = parker.data(parse(self.__chars).getroot()) elements = list(parsed.values()) if len(elements) > 0: elements = elements[0] else: elements = [] for row_number, row in enumerate(elements, start=1): keys, values = zip(*(row.items())) yield (row_number, list(keys), list(values))
def lambda_handler(event, context): data = json.loads(event) if data['isFile']: xml = open(data['xml'], 'r').read() else: xml = data if data['attributes']: result = bf.data(fromstring(xml)) else: result = parker.data(fromstring(xml), preserve_root=True) return json.dumps(result)
def get_data(self, unique_data_ids=None, sample=False, output_type='csv', **kwargs): """ Returns a JSON object of the entire data set. """ data_json = None db = kwargs.get('db', None) if unique_data_ids is None: unique_data_ids = self._available_unique_data_ids for u in unique_data_ids: if (u not in self._available_unique_data_ids): logger.info( " The unique_data_id '{}' is not supported by the DhcdApiConn" .format(u)) else: result = self.get(self._urls[u], params=self._params[u]) if result.status_code != 200: err = "An error occurred during request: status {0}" logger.exception(err.format(result.status_code)) continue data_xml_root = xml_fromstring(result.text) data_xml_records = data_xml_root.findall('record') data_json = xml_to_json.data(data_xml_root) results = [ DhcdResult({e.tag: e.text for e in list(r)}, self._fields[u]).data for r in data_xml_records ] self.result_to_csv(self._fields[u], results, self.output_paths[u]) #Convert to format expected by database if u == 'dhcd_dfd_properties': self.create_project_subsidy_csv('dhcd_dfd_properties', PROJECT_FIELDS_MAP, SUBSIDY_FIELDS_MAP, PROJECT_ADDRE_FIELDS_MAP, db)
def getmonthly(self): r = pr.data(self.elem) for ea in r: if 'monthly' in ea: if type(r[ea]) is list: s = pd.DataFrame() for j in r[ea]: v=pd.DataFrame(j['CustomMonthlyReport']) v['zone'] = j['for'] s=pd.concat([s,v]) s=s.set_index(['zone','name']) self.__setattr__(ea,s) else: pd.DataFrame(r[ea]['CustomMonthlyReport']) self.__setattr__(ea,pd.DataFrame(r[ea]['CustomMonthlyReport']))
def test_nested_empty(): # arrange input = open("./test-data/test_input/nested_empty.xml", "r") input_text = input.read() print("input_text: " + input_text) expected = open("./test-data/expected_output/nested_empty.json", "r") expected_text = expected.read() print("expected_text: " + expected_text) # act res = dumps(parker.data(fromstring(input_text), preserve_root=True)) print("res: " + str(res)) # assert assert res == expected_text
def http_api_get(url): headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36', 'Referer': 'http://iotobservatory.io', 'Content-Type': 'application/json' } try: r = requests.get(url, headers=headers) ret = r.content ret = xml_format(ret) ret_json = json.dumps(parker.data(fromstring(ret))) except Exception as e: print(e) ret_json = {} return ret_json
def format_dmf_response(self, response, ssn): dmf_record = parker.data(fromstring(response.content)) dmf_record_present = False full_name = None if dmf_record['Record'].get('DmfSearch', False): dmf_record_present = True if dmf_record['Record']['CommercialNameSearch'].get('Identity', False): full_name_string = dmf_record['Record']['CommercialNameSearch'][ 'Identity']['NameInfo']['WholeName'] full_name = full_name_string.split(',')[0] return { 'ssn': ssn, 'full_name': full_name, 'dmf_record_present': dmf_record_present }
def get_json_xml(data_folder, num_entries=0): """ Using elemtree, parse xml to json using the parker transformation. The parker transformation keeps the tree structure in the json but removes the attributes. There are however no attributes in the xml. """ i = 0 result_dict = {} for filename in sorted(os.listdir(data_folder)): target_filename = data_folder + filename f = open(target_filename, 'r') text = f.read() xml = fromstring(text) mydict = json.loads(json.dumps(bf.data(xml))) result_dict[filename[0:-4]] = mydict i += 1 if i == num_entries: break return result_dict
def save_to_mongo(content): client = MongoClient() dbtemp = client.temp ## timestmp = datetime.datetime.now().strftime("%Y-%d-%d %H:%M:%S") timestmp = datetime.datetime.utcnow() xmlread = content.decode('ascii') xmlread = fromstring(xmlread) newfile = dumps(parker.data(xmlread)) data = loads(newfile) result = dbtemp.new_req_data.insert_one(data) db = client.reqdata cursor = dbtemp.new_req_data.distinct( "Unit.Packet.Payload.ResultSet.Jobs.Job") for document in cursor: newreq = {"dateAdded": timestmp, "req": document} db.req_data.insert_one(newreq) dbtemp.new_req_data.drop()
def save_to_mongo(content): client = MongoClient() dbtemp = client.temp timestmp = datetime.datetime.now().strftime("%Y-%d-%d %H:%M:%S") xmlread = content.decode('ascii') xmlread = fromstring(xmlread) ## bf = BadgerFish(dict_type=OrderedDict) newfile = dumps(parker.data(xmlread)) data = loads(newfile) ## print (data) result = dbtemp.new_req_data.insert_one(data) ## print("Result: ", result) db = client.reqdata cursor = dbtemp.new_req_data.distinct( "Unit.Packet.Payload.ResultSet.Jobs.Job") for document in cursor: newreq = {"timestamp": timestmp, "req": document} print(newreq) db.req_data.insert_one(newreq)
def save_to_mongo(content): client = MongoClient() ## client = MongoClient("192.168.1.169", 27107) dbtemp = client.temp timestmp = datetime.datetime.utcnow() xmlread = content.decode('ascii') xmlread = fromstring(xmlread) newfile = dumps(parker.data(xmlread)) data = loads(newfile) result = dbtemp.new_req_data.insert_one(data) db=client.reqdata cursor = dbtemp.new_req_data.distinct("Unit.Packet.Payload.ResultSet.Jobs.Job") for document in cursor: newreq = { "dateAdded": timestmp, "req" : document} db.req_data.insert_one(newreq) dbtemp.new_req_data.drop() print("New Reqs Added")