def parseProductXmlAndStore(): tcgOutputFile = open(os.path.join(PROJECT_ROOT, 'tcgoutput.xml')) #print(tcgOutputFile) #print(tcgOutputFile.read()) tcgOutputString = tcgOutputFile.read(); #print(tcgOutputString) tcgXml = ET.fromstring(tcgOutputString) #print tcgXml tcgDictList = [] for child in tcgXml: #print child tcgDictList.append(xmldict.xml_to_dict(child)) #print tcgDictList[1] # tcgOutputDict = xmldict.xml_to_dict(tcgOutputString) # print(tcgOutputDict) # productArray = tcgOutputDict['products']['product'] # print(productArray) # for product in productArray: # print product['id'] productList = [] for tcgDict in tcgDictList: prod = Product.create(tcgDict['product']) productList.append(prod) #print productList[0].tcgId return productList
def Get_Config_State(ip_addr=""):
print("Start function Get_Config_State of IP {0}".format(ip_addr))
file = open("Netconf Credential.txt", "r")
Filter = "<filter><CISCO-CONFIG-MAN-MIB><ccmHistory></ccmHistory><ccmCTIDObjects></ccmCTIDObjects><ccmCLIHistoryCommandTable></ccmCLIHistoryCommandTable></CISCO-CONFIG-MAN-MIB></filter>"
for line in file:
Netconf_Device_Data = line.split()
print(" Data from file :", Netconf_Device_Data)
if (ip_addr == Netconf_Device_Data[0]):
try:
m = manager.connect(host=Netconf_Device_Data[0],
port=Netconf_Device_Data[1],
username=Netconf_Device_Data[2],
password=Netconf_Device_Data[3],
hostkey_verify=False,
device_params={'name': 'default'},
allow_agent=False,
look_for_keys=False)
except:
print(
"****ERROR on Get_Config_State with False : Cannot connect to {0}"
.format(ip_addr))
return False
result = m.get(Filter)
if (result != False):
print("Complete function Get_Config_State")
m.close_session()
return xmldict.xml_to_dict(result.xml)
else:
print(
"****ERROR on Get_Config_State with False : Cannot Get Netconf output from {0}"
.format(ip_addr))
m.close_session()
return False
def get_house_roll_call_vote_data(congress_number):
data = []
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
session_number = 1
while session_number < 3:
roll_number = 1
while True:
try:
url = _get_house_roll_call_vote_data_url(congress_number, session_number, roll_number)
response = opener.open(url)
vote_data = response.read()
# some redirect mechanism has blocked the HTTP error from happenning
# special treatment made using the redirected-page to jump out of the loop
if re.search('<!DOCTYPE html', vote_data):
# usually an XML is rendered, but if the url is not valid
# webpage is redirected to an html page
break
# data.append(_get_formatted_vote_data(vote_data))
dict = xmldict.xml_to_dict(vote_data)
data.append(dict)
roll_number += 1
except urllib2.HTTPError:
print 'HTTP error occurred at ' + url
session_number += 1
return data
def process(self, obj):
if obj.path in self.processed_pages:
return obj
depth = len([
i for i in list(os.path.split(os.path.dirname(obj.path)))
if i != ''
])
if depth < 2:
depthstr = "./"
else:
depthstr = '../' * (depth - 1)
obj.data.add_data(['page', 'root'], depthstr[:-1])
self.processed_pages.append(obj.path)
varxml = base.get_in_containing_tag(obj.text, 'cover')
if varxml is not None:
if varxml[0][0][0] != 0:
self.logger.error(
f'Cover info for "{obj.name}" does not appear at top of file'
)
return obj
xmls = obj.text[varxml[0][0][0]:varxml[1][0][1]]
obj.data.update(xmldict.xml_to_dict(xmls))
obj.text = obj.text.replace(xmls, '', 1)
obj.was_processed = True
return obj
def list_keypairs(self):
''' List the keys, libcloud didn't work so we had to fetch some xml'''
aws_schema = "http://ec2.amazonaws.com/doc/2010-08-31/"
xml_as_dict = xmldict.xml_to_dict(
self.conn.connection.request(self.conn.path,
params={
"Action": "DescribeKeyPairs"
}).__dict__["body"].replace(
aws_schema, ""))
if xml_as_dict["DescribeKeyPairsResponse"]["keySet"] is None:
keypairs = []
else:
keypairs = xml_as_dict["DescribeKeyPairsResponse"]["keySet"][
"item"]
if "keyName" in keypairs:
keypairs["keyMaterial"] = ""
keypairs = [keypairs]
else:
for item in keypairs:
if "keyName" in keypairs:
item["keyMaterial"] = ""
return [{
"public_key": "",
"name": k["keyName"],
"fingerprint": k["keyFingerprint"]
} for k in keypairs]
def test_xml_to_dict_strict(self):
test = '''
<payment_method>
<created_at type="datetime">2011-02-12T20:20:46Z</created_at>
<is_retained type="boolean">true</is_retained>
<messages>
<message class="error" context="input.cvv" key="too_long" />
<message class="error" context="input.card_number" key="failed_checksum" />
</messages>
</payment_method>
'''
expected = {'payment_method': {
'created_at': {
'@type': 'datetime',
'#value': datetime.datetime(2011, 2, 12, 20, 20, 46),
'#text': '2011-02-12T20:20:46Z'
},
'is_retained': {
'@type': 'boolean',
'#value': True,
'#text': 'true'
},
'messages': {'message': [{'@class': 'error',
'@context': 'input.cvv',
'@key': 'too_long'},
{'@class': 'error',
'@context': 'input.card_number',
'@key': 'failed_checksum'}]},
}}
self.assertEqual(expected, xml_to_dict(test, strict=True))
def get_xml_write(list_class, list_b, imagePath, savepath):
with open('template.xml', 'r') as fid:
xml_str = fid.read()
# xml_dict = xmldict.xml_to_dict(xml_str)['annotation']['object']
xml_dict = xmldict.xml_to_dict(xml_str)
del xml_dict['annotation']['object']
objects = []
for i, v in enumerate(list_b):
object = {}
object['name'] = list_class[i]
object['truncated'] = 0
object['difficult'] = 0
object['pose'] = 'Unspecified'
bndbox = {
'xmin': str(v[0]),
'ymin': str(v[1]),
'xmax': str(v[2]),
'ymax': str(v[3])
}
object['bndbox'] = bndbox
objects.append(object)
xml_dict['annotation']['object'] = objects
dict_xml = xmldict.dict_to_xml(xml_dict)
dom = parseString(dict_xml).toprettyxml()
# print(xml_dict['annotation']['folder'])
with open(savepath + '\\' + imagePath.replace('jpg', 'xml'), 'w') as f:
f.write(dom)
f.close()
def queryStation(sID):
api_key = "UPIIJBD0QEG"
url = "http://api.rideuta.com/SIRI/SIRI.svc/"
url += "StopMonitor?stopid=" # Query type
url += sID # Stop ID
url += "&minutesout=20" # How far in the future to query
url += "&onwardcalls=false" # Include vehicle calls inbetween current location and stop
url += "&filterroute=" # Filter vehicles
url += "&usertoken="
url += api_key
r = requests.get(url)
xml = r.content
d = xmldict.xml_to_dict(xml)
d2 = d['{http://www.siri.org.uk/siri}Siri']['{http://www.siri.org.uk/siri}StopMonitoringDelivery']['{http://www.siri.org.uk/siri}MonitoredStopVisit']['{http://www.siri.org.uk/siri}MonitoredVehicleJourney']
if '{http://www.siri.org.uk/siri}MonitoredVehicleJourney' not in d['{http://www.siri.org.uk/siri}Siri']['{http://www.siri.org.uk/siri}StopMonitoringDelivery']['{http://www.siri.org.uk/siri}MonitoredStopVisit']:
print "Uh Oh!"
else:
# Print Query Header
localtime = time.asctime(time.localtime(time.time()))
print ("\nQUERY STATION: %s \n" % colored.black(d['{http://www.siri.org.uk/siri}Siri']['{http://www.siri.org.uk/siri}StopMonitoringDelivery']['{http://www.siri.org.uk/siri}Extensions']['{http://www.siri.org.uk/siri}StopName']))
print "Query Time: %s /n" % str(localtime)
getData(nStopID, 0)
getData(sStopID, 1)
def test_xml_to_dict_strict(self):
test = '''
<payment_method>
<created_at type="datetime">2011-02-12T20:20:46Z</created_at>
<is_retained type="boolean">true</is_retained>
<messages>
<message class="error" context="input.cvv" key="too_long" />
<message class="error" context="input.card_number" key="failed_checksum" />
</messages>
</payment_method>
'''
expected = {'payment_method': {
'created_at': {
'@type': 'datetime',
'#value': datetime.datetime(2011, 2, 12, 20, 20, 46),
'#text': '2011-02-12T20:20:46Z'
},
'is_retained': {
'@type': 'boolean',
'#value': True,
'#text': 'true'
},
'messages': {'message': [{'@class': 'error',
'@context': 'input.cvv',
'@key': 'too_long'},
{'@class': 'error',
'@context': 'input.card_number',
'@key': 'failed_checksum'}]},
}}
self.assertEqual(expected, xml_to_dict(test, strict=True))
def get_senate_roll_call_vote(congress_number):
# compare
web_structure_has_changed = _check_web_structure(_get_senate_roll_call_vote_url(congress_number, 1, 1),
WEB_STRUCTURE_FINGER_PRINT)
if web_structure_has_changed:
print 'the web structure has changed, we need to modify our code to adapt to the changes'
return [None, None]
# the function is meant to return the senate vote data as an array of dictionaries
# The index of a specific vote data is vote_number - 1 due to zero based indexing system
dictionaries = []
# this opener adds a header to the HTTP requests, and mimic the behavior of a browser
# in order to avoid 503 service not available error
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
# each congress has no more than 2 sessions
session_number = 1
while session_number < 3:
# re-basing the vote number
vote_number = 1
while True:
try:
url = _get_senate_roll_call_vote_url(congress_number, session_number, vote_number)
# print url
# dictionary=xmldict.xml_to_dict(requests.get(url).content)
response = opener.open(url)
dictionary = xmldict.xml_to_dict(response.read())
internal_ids = _get_internal_ids(dictionary)
dictionary['ndp_specs'] = {'internal_id': internal_ids[0], 'amendment_to_id': internal_ids[1]}
vote_number += 1
dictionaries.append(dictionary)
except urllib2.HTTPError:
# either the vote number has incremented high enough, or some unknown url error occurred
break
session_number += 1
return dictionaries
def get_xml_write2(classname, box, savepath):
with open('template.xml', 'r') as fid:
xml_str = fid.read()
# xml_dict = xmldict.xml_to_dict(xml_str)['annotation']['object']
xml_dict = xmldict.xml_to_dict(xml_str)
del xml_dict['annotation']['object']
objects = []
for i in box:
object = {}
object['name'] = classname
object['truncated'] = 0
object['difficult'] = 0
object['pose'] = 'Unspecified'
bndbox = {
'xmin': str(i[0]),
'ymin': str(i[1]),
'xmax': str(i[2]),
'ymax': str(i[3])
}
object['bndbox'] = bndbox
objects.append(object)
# [x, y, x + w, y + h]
xml_dict['annotation']['object'] = objects
dict_xml = xmldict.dict_to_xml(xml_dict)
dom = parseString(dict_xml).toprettyxml()
with open(savepath, 'w') as f:
f.write(dom)
f.close()
def set_xml(self, input):
result = xml_to_dict(input)
for k, v in result.iteritems():
if k != 'id' and k in self.getFields():
setattr(self, k, v)
elif k == 'id':
self._id = v
def parse_vpn_response(response, mark):
config = OrderedDict()
for connection in response['VpnConnections']:
vpn_id = connection['VpnConnectionId']
config[vpn_id] = OrderedDict()
vpn_config = xml_to_dict(connection['CustomerGatewayConfiguration'])
for index, tunnel in enumerate(
vpn_config['vpn_connection']['ipsec_tunnel']):
tunnel_key = 'tunnel-%s' % index
config[vpn_id][tunnel_key] = OrderedDict()
config[vpn_id][tunnel_key]['right_inside_ip'] = '%s/%s' % (
tunnel['customer_gateway']['tunnel_inside_address']
['ip_address'], tunnel['customer_gateway']
['tunnel_inside_address']['network_cidr'])
config[vpn_id][tunnel_key]['right_outside_ip'] = tunnel[
'customer_gateway']['tunnel_outside_address']['ip_address']
config[vpn_id][tunnel_key]['right_asn'] = tunnel[
'customer_gateway']['bgp']['asn']
config[vpn_id][tunnel_key]['left_inside_ip'] = '%s/%s' % (
tunnel['vpn_gateway']['tunnel_inside_address']['ip_address'],
tunnel['vpn_gateway']['tunnel_inside_address']['network_cidr'])
config[vpn_id][tunnel_key]['left_outside_ip'] = tunnel[
'vpn_gateway']['tunnel_outside_address']['ip_address']
config[vpn_id][tunnel_key]['left_asn'] = tunnel['vpn_gateway'][
'bgp']['asn']
config[vpn_id][tunnel_key]['psk'] = tunnel['ike']['pre_shared_key']
config[vpn_id][tunnel_key]['mark'] = next(mark)
return config
def test_xml_to_dict_strict(self):
test = """
<payment_method>
<created_at type="datetime">2011-02-12T20:20:46Z</created_at>
<is_retained type="boolean">true</is_retained>
<messages>
<message class="error" context="input.cvv" key="too_long" />
<message class="error" context="input.card_number" key="failed_checksum" />
</messages>
</payment_method>
"""
expected = {
"payment_method": {
"created_at": {
"@type": "datetime",
"#value": datetime.datetime(2011, 2, 12, 20, 20, 46),
"#text": "2011-02-12T20:20:46Z",
},
"is_retained": {"@type": "boolean", "#value": True, "#text": "true"},
"messages": {
"message": [
{"@class": "error", "@context": "input.cvv", "@key": "too_long"},
{"@class": "error", "@context": "input.card_number", "@key": "failed_checksum"},
]
},
}
}
self.assertEqual(expected, xml_to_dict(test, strict=True))
def upload_metadata(self, project, raw_metadata):
''' upload xml or json to the metadata server'''
headers = {}
headers.update(self.headers)
try:
file_type = magic.from_buffer(raw_metadata[:200])
if file_type == "XML document text":
#headers["content-type"] = "xml"
metadata = xmldict.xml_to_dict(
raw_metadata)["ResultSet"]["Result"]
except Exception:
metadata = raw_metadata
#else:
# metadata = json.loads(raw_metadata)
path = "%s%s%s" % (self.id_service, self.METADATA_PATH, project)
with SourceInterface(self.interface):
chunk_size = 1024
ids = []
for start in range(0, len(metadata), chunk_size):
response = requests.put(path,
data=json.dumps(metadata[start:start +
chunk_size]),
headers=headers)
if "x-id-auth-token" in response.headers:
self.headers["x-id-auth-token"] = response.headers[
"x-id-auth-token"]
ids.append(response.text)
return ids
def response_to_dict(self,response,return_format):
response_dict = {}
json_match = re.search('^\s*{', response)
xml_match = re.search('^\s*\<', response)
ini_match = re.search('^\s*\[', response)
if json_match :
self.log.info(" Response is in 'JSON' format and Converting to '"+return_format+"' format")
# Formatting the json string
response = re.sub(r"{\s*'?(\w)", r'{"\1', response)
response = re.sub(r",\s*'?(\w)", r',"\1', response)
response = re.sub(r"(\w)'?\s*:", r'\1":', response)
response = re.sub(r":\s*'(\w)'\s*([,}])", r':"\1"\2', response)
try :
import json
response_dict = json.loads(response)
except StandardError:
self.log.exception( "Json Parser is unable to parse the string" )
return response_dict
elif ini_match :
self.log.info(" Response is in 'INI' format and Converting to '"+return_format+"' format")
from configobj import ConfigObj
response_file = open("respnse_file.temp",'w')
response_file.write(response)
response_file.close()
response_dict = ConfigObj("respnse_file.temp")
return response_dict
elif xml_match :
self.log.info(" Response is in 'XML' format and Converting to '"+return_format+"' format")
try :
response_dict = xmldict.xml_to_dict("<response> "+str(response)+" </response>")
except StandardError:
self.log.exception()
return response_dict
def list_keys(self):
""" List the keys, libcloud didn't work so we had to fetch some xml"""
aws_schema = "http://ec2.amazonaws.com/doc/2010-08-31/"
xml_as_dict = xmldict.xml_to_dict(
self.conn.connection.request(self.conn.path, params={"Action": "DescribeKeyPairs"})
.__dict__["body"]
.replace(aws_schema, "")
)
if xml_as_dict["DescribeKeyPairsResponse"]["keySet"] is None:
return "[]"
else:
result = xml_as_dict["DescribeKeyPairsResponse"]["keySet"]["item"]
if "keyName" in result:
result["keyMaterial"] = ""
result = [result]
else:
for item in result:
if "keyName" in result:
item["keyMaterial"] = ""
if self.options.id:
result = keypair_by_name(self.options.id, result)
keys_json = json.dumps(result)
return keys_json
def get_xml_write1(classname, box, imagePath, savepath):
with open('template.xml', 'r') as fid:
xml_str = fid.read()
# xml_dict = xmldict.xml_to_dict(xml_str)['annotation']['object']
xml_dict = xmldict.xml_to_dict(xml_str)
del xml_dict['annotation']['object']
object = {}
object['name'] = classname
object['truncated'] = 0
object['difficult'] = 0
object['pose'] = 'Unspecified'
bndbox = {
'xmin': str(box[0]),
'ymin': str(box[1]),
'xmax': str(box[0] + box[2]),
'ymax': str(box[1] + box[3])
}
object['bndbox'] = bndbox
# [x, y, x + w, y + h]
xml_dict['annotation']['object'] = object
dict_xml = xmldict.dict_to_xml(xml_dict)
dom = parseString(dict_xml).toprettyxml()
with open(savepath + '\\' + imagePath.replace('jpg', 'xml'), 'w') as f:
f.write(dom)
f.close()
def set_xml(self, input):
result = xml_to_dict(input)
for k, v in result.iteritems():
if k != 'id' and k in self.getFields():
setattr(self, k, v)
elif k == 'id':
self._id = v
def upload_metadata(self, project, raw_metadata):
''' upload xml or json to the metadata server'''
headers = {}
headers.update(self.headers)
try:
file_type = magic.from_buffer(raw_metadata[:200])
if file_type == "XML document text":
#headers["content-type"] = "xml"
metadata = xmldict.xml_to_dict(raw_metadata)["ResultSet"]["Result"]
except Exception:
metadata = raw_metadata
#else:
# metadata = json.loads(raw_metadata)
path = "%s%s%s" % (self.id_service, self.METADATA_PATH, project)
with SourceInterface(self.interface):
chunk_size = 1024
ids = []
for start in range(0, len(metadata), chunk_size):
response = requests.put(path, data=json.dumps(
metadata[start: start + chunk_size]), headers=headers)
if "x-id-auth-token" in response.headers:
self.headers["x-id-auth-token"] = response.headers[
"x-id-auth-token"]
ids.append(response.text)
return ids
def convertXmlToDict(self):
'''
'''
ret_xmlDict = {}
with open(self.file_name, 'r') as content_file:
content = content_file.read()
ret_xmlDict = xmldict.xml_to_dict(content)
return ret_xmlDict
def extract_xml_index(file, prot_set, initial_id_list):
with open(file) as fd:
tmp_set = set()
doc = xml_to_dict(fd.read())
prot_list = doc["musite"]["protein-list"]["protein"]
for p in prot_list:
tmp_set.add(p["accession"])
return prot_set.union(tmp_set) - initial_id_list
def get_job_config(server):
conf_xml = server.get_job_config('api_test_framework')
conf_xml = conf_xml.split("\n", 1)[1]
print conf_xml
conf = xmldict.xml_to_dict(str(conf_xml))
import pprint
pprint.pprint(conf)
def read(self):
result = {}
tree = ET.tostring( self.xml[ len(self.xml) - 1 ] )
result['deputado'] = xmldict.xml_to_dict(tree)
return result
def getFeed(self):
data = []
for podcast in self.opt['podcasts']:
get = urllib2.urlopen(podcast).read()
xml = xmldict.xml_to_dict(get)
data += xml['rss']['channel']['item']
return data
def test_xml_to_dict(self):
test = '''
<payment_method>
<payment_method_token>QhLaMNNpvHwfnFbHbUYhNxadx4C</payment_method_token>
<created_at type="datetime">2011-02-12T20:20:46Z</created_at>
<updated_at type="datetime">2011-04-22T17:57:30Z</updated_at>
<custom>Any value you want us to save with this payment method.</custom>
<is_retained type="boolean">true</is_retained>
<is_redacted type="boolean">false</is_redacted>
<is_sensitive_data_valid type="boolean">false</is_sensitive_data_valid>
<messages>
<message class="error" context="input.cvv" key="too_long" />
<message class="error" context="input.card_number" key="failed_checksum" />
</messages>
<last_four_digits>1111</last_four_digits>
<card_type>visa</card_type>
<first_name>Bob</first_name>
<last_name>Smith</last_name>
<expiry_month type="integer">1</expiry_month>
<expiry_year type="integer">2020</expiry_year>
<address_1 nil="true"></address_1>
<address_2 nil="true"></address_2>
<city nil="true"></city>
<state nil="true"></state>
<zip nil="true"></zip>
<country nil="true"></country>
</payment_method>
'''
expected = {'payment_method': {'address_1': {'nil': 'true'},
'address_2': {'nil': 'true'},
'card_type': 'visa',
'city': {'nil': 'true'},
'country': {'nil': 'true'},
'created_at': datetime.datetime(2011, 2, 12, 20, 20, 46),
'custom': 'Any value you want us to save with this payment method.',
'expiry_month': 1,
'expiry_year': 2020,
'first_name': 'Bob',
'is_redacted': False,
'is_retained': True,
'is_sensitive_data_valid': False,
'last_four_digits': '1111',
'last_name': 'Smith',
'messages': {'message': [{'class': 'error',
'context': 'input.cvv',
'key': 'too_long'},
{'class': 'error',
'context': 'input.card_number',
'key': 'failed_checksum'}]},
'payment_method_token': 'QhLaMNNpvHwfnFbHbUYhNxadx4C',
'state': {'nil': 'true'},
'updated_at': datetime.datetime(2011, 4, 22, 17, 57, 30),
'zip': {'nil': 'true'}}}
self.assertEqual(expected, xml_to_dict(test))
def convert():
'''Loads the cards.xml file, converts to JSON, and saves it'''
#Load the cards file, make sure it exists
try:
cards = open('cards.xml')
except IOError:
print 'No cards.xml file defined! Make sure cards.xml exists'
return False
#Read the file
cards = cards.read()
#Convert the sucker
cards_dict = xmldict.xml_to_dict(cards)
#Prepare json file
json_file = open('cards.json', 'w')
#Get the cards
cards = cards_dict['cockatrice_carddatabase']['cards']['card']
#MongoDB expects the JSON file to have one document per line, so we need to
# iterate over each line in the file
for card in cards:
#Lets add some extra info to the card object
# Store power and toughness separately as ints
i = 0
for type in ['power', 'toughness']:
try:
card[type] = card['pt'].split('/')[i]
try:
#try to turn to int. It may not be an int
# (in case of X)
# pt is always X/Y, so split on / and use 0 index
# for power and 1 for toughness
card[type] = int(card['pt'].split('/')[i])
except ValueError:
pass
except (KeyError, IndexError) as e:
#card has no power or toughness
pass
#keep track of what item we're on
i = i + 1
#Also save the set name as a key so we don't have to access the set
# key to get the set text
card['setText'] = card['set']['#text']
#Save it
json_file.write(json.dumps(card) + '\n')
#close it
json_file.close()
return True
def convert():
'''Loads the cards.xml file, converts to JSON, and saves it'''
#Load the cards file, make sure it exists
try:
cards = open('cards.xml')
except IOError:
print 'No cards.xml file defined! Make sure cards.xml exists'
return False
#Read the file
cards = cards.read()
#Convert the sucker
cards_dict = xmldict.xml_to_dict(cards)
#Prepare json file
json_file = open('cards.json', 'w')
#Get the cards
cards = cards_dict['cockatrice_carddatabase']['cards']['card']
#MongoDB expects the JSON file to have one document per line, so we need to
# iterate over each line in the file
for card in cards:
#Lets add some extra info to the card object
# Store power and toughness separately as ints
i=0
for type in ['power', 'toughness']:
try:
card[type] = card['pt'].split('/')[i]
try:
#try to turn to int. It may not be an int
# (in case of X)
# pt is always X/Y, so split on / and use 0 index
# for power and 1 for toughness
card[type] = int(card['pt'].split('/')[i])
except ValueError:
pass
except (KeyError, IndexError) as e:
#card has no power or toughness
pass
#keep track of what item we're on
i = i+1
#Also save the set name as a key so we don't have to access the set
# key to get the set text
card['setText'] = card['set']['#text']
#Save it
json_file.write(json.dumps(card) + '\n')
#close it
json_file.close()
return True
def test_xml_to_dict_list(self):
test = """
<messages>
<message>message1</message>
<message>message2</message>
</messages>
"""
expected = {"messages": {"message": ["message1", "message2"]}}
self.assertEqual(expected, xml_to_dict(test, strict=False))
def test_xml_to_dict_list(self):
test = '''
<messages>
<message>message1</message>
<message>message2</message>
</messages>
'''
expected = {'messages': {'message': ['message1', 'message2']}}
self.assertEqual(expected, xml_to_dict(test, strict=False))
def test_xml_to_dict_list(self):
test = '''
<messages>
<message>message1</message>
<message>message2</message>
</messages>
'''
expected = {'messages': {'message': ['message1', 'message2']}}
self.assertEqual(expected, xml_to_dict(test, strict=False))
def test_xml_to_dict_namespace(self):
xml_str1 = """<root id="1" xmlns="somenamespace"><items><item>1</item><item>2</item></items></root>"""
xml_str2 = """<root id="1"><items><item>1</item><item>2</item></items></root>"""
expected1 = {'{somenamespace}root': {'{somenamespace}items': {'{somenamespace}item': ['1', '2']}}}
expected2 = {'root': {'items': {'item': ['1', '2']}}}
self.assertEqual(expected1, xml_to_dict(xml_str1, strict=False))
self.assertEqual(expected1, xml_to_dict(xml_str1))
self.assertEqual(expected2, xml_to_dict(xml_str1, False, True))
self.assertEqual(expected2, xml_to_dict(xml_str1, remove_namespace=True))
self.assertEqual(expected2, xml_to_dict(xml_str2, strict=False))
self.assertEqual(expected2, xml_to_dict(xml_str2))
self.assertEqual(expected2, xml_to_dict(xml_str2, False, True))
self.assertEqual(expected2, xml_to_dict(xml_str2, remove_namespace=True))
def read_xml(path):
content = ""
data = {}
with open(path, "r") as f:
content = f.read()
if content:
data = xmldict.xml_to_dict(content)
return data
def main():
"""
Simple main method calling our function.
"""
interfaces = get_Netconf_result(FILE)
temp = xml.dom.minidom.parseString(interfaces.xml).toprettyxml()
file = open("Netconf_result.xml", "w")
file.write(temp)
dict1 = xmldict.xml_to_dict(interfaces.xml)
file = open("Netconf_result.json", "w")
file.write(json.dumps(dict1, indent=4))
def configparser(self):
'''
It will parse the config file (teston.cfg) and return as dictionary
'''
matchFileName = re.match(r'(.*)\.cfg', self.configFile, re.M | re.I)
if matchFileName:
xml = open(self.configFile).read()
try :
self.configDict = xmldict.xml_to_dict(xml)
return self.configDict
except Exception:
print "There is no such file to parse " + self.configFile
def configparser(self):
'''
It will parse the config file (teston.cfg) and return as dictionary
'''
matchFileName = re.match(r'(.*)\.cfg', self.configFile, re.M | re.I)
if matchFileName:
xml = open(self.configFile).read()
try:
self.configDict = xmldict.xml_to_dict(xml)
return self.configDict
except:
print "There is no such file to parse " + self.configFile
def put_metadata(project_name):
''' Return info about the cloud '''
if request.headers["content-type"] in ["xml", "text/xml",
"application/xml"]:
raw_metadata = xmldict.xml_to_dict(request.data)
elif request.headers["content-type"] in ["json", "text/json",
"application/json"]:
raw_metadata = json.loads(request.data)
host, port = request.host.split(":")
metadata_service = "http://%s:%s" % (socket.gethostbyaddr(host)[0], port)
return update_metadata(g.user, project_name, raw_metadata, metadata_service,
_create_id())
def getConcepts(text):
# Set params for keyword search
params = AlchemyAPI.AlchemyAPI_ConceptParams()
params.setMaxRetrieve(30)
if isinstance(text, unicode):
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore')
result = xml_to_dict(alchemyObj.TextGetRankedConcepts(text, params))
keywords = result['results']['concepts']['concept']
temp = []
for result in keywords:
temp.append({'relevance': result['relevance'], 'text': result['text']})
keywords = temp
return keywords
def getKeywords(text):
# Set params for keyword search
params = AlchemyAPI.AlchemyAPI_KeywordParams()
params.setMaxRetrieve(30)
params.setKeywordExtractMode('strict')
if isinstance(text, unicode):
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore')
result = xml_to_dict(alchemyObj.TextGetRankedKeywords(text, params))
try:
keywords = result['results']['keywords']['keyword']
except:
keywords = []
return keywords
def process(self, obj):
super().process(obj)
varxml = get_in_containing_tag(obj.text, 'defaults')
if varxml is not None:
if varxml[0][0][0] != 0:
self.logger.error(
f'Default variables for "{obj.name}" do not appear at top of file'
)
return obj
xmls = obj.text[varxml[0][0][0]:varxml[1][0][1]]
obj.variables = xmldict.xml_to_dict(xmls)['defaults']
obj.text = obj.text.replace(xmls, '', 1)
obj.processed = True
return obj
def convert():
'''Loads the posts.xml file, converts to JSON, and saves it'''
#Load the posts file, make sure it exists
try:
notes = open('mysite_notes.xml')
except IOError:
print 'No posts.xml file defined! Make sure .xml exists'
return False
#Read the file
notes = notes.read()
#Convert the sucker
notes_dict = xmldict.xml_to_dict(notes)
#Prepare json file
json_file = open('notes.json', 'w')
#Get the notes
notes = notes_dict['database']['table']
print '*' * 42
print notes
for item in notes:
print '*' * 42
print 'NOTE ITEM:'
print item
#MongoDB expects the JSON file to have one document per line, so we need to
# iterate over each line in the file
for card in notes:
#Create an object from the existing one
temp_note = {}
temp_note['slug'] = card['column'][0]['#text']
temp_note['title'] = card['column'][1]['#text']
temp_note['post_date'] = card['column'][2]['#text']
temp_note['post_last_edit_date'] = card['column'][3]['#text']
temp_note['category'] = card['column'][6]['#text']
temp_note['content'] = card['column'][7]['#text']
temp_note['description'] = card['column'][8]['#text']
#Save it
json_file.write(json.dumps(temp_note) + '\n')
#close it
json_file.close()
return True
def parse(self,fileName) :
'''
This will parse the params or topo or cfg file and return content in the file as Dictionary
'''
self.fileName = fileName
matchFileName = re.match(r'(.*)\.(params|topo|cfg)', self.fileName, re.M | re.I)
if matchFileName:
xml = open(fileName).read()
try :
parsedInfo = xmldict.xml_to_dict(xml)
return parsedInfo
except Exception as e:
print "Error parsing file " + fileName + ": " + e.message
else :
print "File name is not correct"
def parse(self, fileName):
'''
This will parse the params or topo or cfg file and return content in the file as Dictionary
'''
self.fileName = fileName
matchFileName = re.match(r'(.*)\.(params|topo|cfg)', self.fileName,
re.M | re.I)
if matchFileName:
xml = open(fileName).read()
try:
parsedInfo = xmldict.xml_to_dict(xml)
return parsedInfo
except StandardError as e:
print "Error parsing file " + fileName + ": " + e.message
else:
print "File name is not correct"
def from_string(self, str_data):
load_dict = xmldict.xml_to_dict(
str_data
).get('root', {})
result_dict = {}
for k, v in load_dict.viewitems():
if isinstance(v, dict) and [self.ITEM_TAG] == v.keys():
v = v.values()[0]
result_dict[k] = v
for k, v in result_dict.viewitems():
if v in ['true', 'false']:
result_dict[k] = v == 'true'
return result_dict
def put_metadata(project_name):
''' Return info about the cloud '''
if request.headers["content-type"] in [
"xml", "text/xml", "application/xml"
]:
raw_metadata = xmldict.xml_to_dict(request.data)
elif request.headers["content-type"] in [
"json", "text/json", "application/json"
]:
raw_metadata = json.loads(request.data)
host, port = request.host.split(":")
metadata_service = "http://%s:%s" % (socket.gethostbyaddr(host)[0], port)
return update_metadata(g.user, project_name, raw_metadata,
metadata_service, _create_id())
def parse(self, fileName):
'''
This will parse the params or topo or cfg file and return content in the file as Dictionary
'''
self.fileName = fileName
matchFileName = re.match(r'(.*)\.(params|topo|cfg)', self.fileName,
re.M | re.I)
if matchFileName:
xml = open(fileName).read()
try:
parsedInfo = xmldict.xml_to_dict(xml)
return parsedInfo
except Exception:
print "There is no such file to parse " + fileName
else:
print "file name is not correct"
def obj_from_result_xml(result):
parsed = xmldict.xml_to_dict(result)
status = True if parsed['ocs']['meta']['status'] == 'ok' else False
r = Result(status,
int(parsed['ocs']['meta']['statuscode']),
parsed['ocs']['meta']['message'])
data = []
for type_, values in parsed['ocs']['data'].items():
convertor = getDecoder(type_, 'xml')
if isinstance(values, list):
for i in values:
data.append(convertor(i))
else:
data.append(convertor(values))
r.data = data
return r
def logout(self):
if self.__session_id:
request = urllib2.Request(self.__base_url +
'LogOut;JSESSIONID=%s?source=%s' %
(self.__session_id, self.__source_id))
response = urllib2.urlopen(request)
result = xmldict.xml_to_dict(ElementTree.XML(
response.read()))['amtd']
if result['result'] == 'LoggedOut':
self.__msg("Logout Success")
self.__session_id = None
self.logged_in = False
else:
self.__msg("Logout Failure")
else:
self.__msg("Repeat Logout")
def test_xml_to_dict_simple(self):
test = '''
<payment_method>
<created_at type="datetime">2011-02-12T20:20:46Z</created_at>
<is_retained type="boolean">true</is_retained>
<first_name>Bob</first_name>
<expiry_month type="integer">1</expiry_month>
<address_1 nil="true"></address_1>
</payment_method>
'''
expected = {'payment_method': {'address_1': {'nil': 'true'},
'created_at': datetime.datetime(2011, 2, 12, 20, 20, 46),
'expiry_month': 1,
'first_name': 'Bob',
'is_retained': True}}
self.assertEqual(expected, xml_to_dict(test, strict=False))
def test_xml_to_dict_simple(self):
test = '''
<payment_method>
<created_at type="datetime">2011-02-12T20:20:46Z</created_at>
<is_retained type="boolean">true</is_retained>
<first_name>Bob</first_name>
<expiry_month type="integer">1</expiry_month>
<address_1 nil="true"></address_1>
</payment_method>
'''
expected = {'payment_method': {'address_1': {'nil': 'true'},
'created_at': datetime.datetime(2011, 2, 12, 20, 20, 46),
'expiry_month': 1,
'first_name': 'Bob',
'is_retained': True}}
self.assertEqual(expected, xml_to_dict(test, strict=False))
def test_xml_to_dict_and_reverse(self):
test = '''<messages>
<message id="1">
a
b
...
</message>
<message id="2">
c
d
...
</message>
</messages>'''
expected = {
'messages': {
'message': [
{
'@id': '1',
'#value': 'a\nb\n...',
'#text': '\na\nb\n...\n',
},
{
'@id': '2',
'#value': 'c\nd\n...',
'#text': '\nc\nd\n...\n',
},
]
}
}
self.assertEqual(expected, xml_to_dict(test, strict=True))
# once converted to dict, go back to xml (do not care about extra blanks)
def _remove_blanks(content):
from lxml import etree
parser = etree.XMLParser(remove_blank_text=True)
elem = etree.XML(content, parser=parser)
return etree.tostring(elem)
self.assertEqual(_remove_blanks(test), dict_to_xml(expected))
def test_xml_to_dict_namespace(self):
xml_str1 = """<root id="1" xmlns="somenamespace"><items><item>1</item><item>2</item></items></root>"""
xml_str2 = """<root id="1"><items><item>1</item><item>2</item></items></root>"""
expected1 = {
'{somenamespace}root': {
'{somenamespace}items': {
'{somenamespace}item': ['1', '2']
}
}
}
expected2 = {'root': {'items': {'item': ['1', '2']}}}
self.assertEqual(expected1, xml_to_dict(xml_str1, strict=False))
self.assertEqual(expected1, xml_to_dict(xml_str1))
self.assertEqual(expected2, xml_to_dict(xml_str1, False, True))
self.assertEqual(expected2, xml_to_dict(xml_str1,
remove_namespace=True))
self.assertEqual(expected2, xml_to_dict(xml_str2, strict=False))
self.assertEqual(expected2, xml_to_dict(xml_str2))
self.assertEqual(expected2, xml_to_dict(xml_str2, False, True))
self.assertEqual(expected2, xml_to_dict(xml_str2,
remove_namespace=True))
def productFromXmlString(tcgXmlString, cardName):
tcgXml = None
try:
tcgXml = ET.fromstring(tcgXmlString)
except:
print('tcgXml Failed to Parse: ' + tcgXmlString)
if(None != tcgXml):
tcgDictList = []
for child in tcgXml:
tcgDictList.append(xmldict.xml_to_dict(child))
tcgDict = get_first(tcgDictList)
product = None
if(None != tcgDict):
productDict = tcgDict['product']
productDict['name'] = cardName
print('tcgProduct: ' + str(productDict))
product = Product.create(productDict)
return product
else:
return None
def xml_filename_to_dict(filename):
"""
Reads the contents of an XML file, parses it and returns a dictionary.
Args:
filename: The name of the file with xml contents.
Returns:
A dictionary with the contents of the xml file.
>>> xml_filename_to_dict('file-1.xml')
{'foo': 'bar'}
"""
# Step 1: Read the contents of the file into a string
f = open(filename).readlines()
fstring = '\n'.join(f)
# Step 2: Convert the contents of the XML file into a dictionary
return xmldict.xml_to_dict(fstring)
def xml_filename_to_dict(filename):
"""
Reads the contents of an XML file, parses it and returns a dictionary.
Args:
filename: The name of the file with xml contents.
Returns:
A dictionary with the contents of the xml file.
>>> xml_filename_to_dict('file-1.xml')
{'foo': 'bar'}
"""
# Step 1: Read the contents of the file into a string
f = open(filename).readlines()
fstring = '\n'.join(f)
# Step 2: Convert the contents of the XML file into a dictionary
return xmldict.xml_to_dict(fstring)
def test_xml_to_dict_and_reverse(self):
test = '''<messages>
<message id="1">
a
b
...
</message>
<message id="2">
c
d
...
</message>
</messages>'''
expected = {'messages':
{'message':
[
{'@id': '1',
'#value': 'a\nb\n...',
'#text': '\na\nb\n...\n',
},
{'@id': '2',
'#value': 'c\nd\n...',
'#text': '\nc\nd\n...\n',
},
]
}
}
self.assertEqual(expected, xml_to_dict(test, strict=True))
# once converted to dict, go back to xml (do not care about extra blanks)
def _remove_blanks(content):
from lxml import etree
parser = etree.XMLParser(remove_blank_text=True)
elem = etree.XML(content, parser=parser)
return etree.tostring(elem)
self.assertEqual(_remove_blanks(test), dict_to_xml(expected))
def _get_formatted_vote_data(data):
# input data should be a string
data = xmldict.xml_to_dict(data)
dict = {}
dict['members'] = {}
dict['members']['member'] = []
votes = data['rollcall-vote']['vote-data']['recorded-vote']
for vote in votes:
member = {}
# with the data we have, I cannot figure out the first name
member['last_name'] = vote['legislator']['@unaccented-name']
member['vote_cast'] = vote['vote']
print vote['vote']
member['state'] = vote['legislator']['@state']
member['party'] = vote['legislator']['@party']
# name-id?
# member['lis_member_id']
# Akaka (D-HI) full
member['member_full'] = member['last_name'] + ' (' + member['party'] + '-' + member['state'] + ')'
dict['members']['member'].append(member)
return dict
def login(self, username, password):
if self.__session_id == None:
params = {
'userid': username,
'password': password,
'source': self.__source_id,
'version': self.__version
}
request = urllib2.Request(self.__base_url + 'LogIn')
request.add_header(
"Content-Type",
"application/x-www-form-urlencoded;charset=utf-8")
try:
response = urllib2.urlopen(request,
urlencode(params).encode('utf-8'))
except urllib2.URLError:
self.__msg("HTTP Failure")
else:
result = xmldict.xml_to_dict(ElementTree.XML(
response.read()))['amtd']
if result['result'] == 'OK' and 'xml-log-in' in result:
self.__msg("Login Success")
self.__session_id = result['xml-log-in']['session-id']
self.logged_in = True
for info_type in result['xml-log-in']:
if (info_type not in [
'accounts', 'cdi', 'associated-account-id'
]):
print info_type, result['xml-log-in'][info_type]
print('\n')
return result['xml-log-in']
else:
self.__msg("Login Failure")
else:
self.__msg("Repeat Login")
def response_to_dict(self, response, return_format):
response_dict = {}
json_match = re.search('^\s*{', response)
xml_match = re.search('^\s*\<', response)
ini_match = re.search('^\s*\[', response)
if json_match:
self.log.info("Response is in 'JSON' format, converting to '" +
return_format + "' format")
# Formatting the json string
response = re.sub(r"{\s*'?(\w)", r'{"\1', response)
response = re.sub(r",\s*'?(\w)", r',"\1', response)
response = re.sub(r"(\w)'?\s*:", r'\1":', response)
response = re.sub(r":\s*'(\w)'\s*([,}])", r':"\1"\2', response)
try:
import json
response_dict = json.loads(response)
except StandardError:
self.log.exception("Json Parser is unable to parse the string")
return response_dict
elif ini_match:
self.log.info("Response is in 'INI' format, converting to '" +
return_format + "' format")
from configobj import ConfigObj
response_file = open("respnse_file.temp", 'w')
response_file.write(response)
response_file.close()
response_dict = ConfigObj("respnse_file.temp")
return response_dict
elif xml_match:
self.log.info("Response is in 'XML' format, converting to '" +
return_format + "' format")
try:
response_dict = xmldict.xml_to_dict("<response> " +
str(response) +
" </response>")
except StandardError:
self.log.exception()
return response_dict
def __pull_options_chains(self, symbol, type, expiry):
# save XML response to file
def _save_to_file(path, raw_xml, expiry):
loc = path + expiry + '.txt'
file = open(loc, 'w')
file.write(raw_xml)
file.close()
if self.__session_id:
request = urllib2.Request(self.__options_url + 'OptionChain?source=%s&symbol=%s&type=%s&range=ALL&expire=%s"es=true' \
% (self.__source_id, symbol, type, expiry))
self.__add_session_cookie(request)
response = urllib2.urlopen(request)
raw_xml = response.read()
# _save_to_file('Drive:\\Local\\Path\\Here\\', raw_xml, expiry)
try:
chains_in_month = xmldict.xml_to_dict(
ElementTree.XML(raw_xml))['amtd']['option-chain-results']
except KeyError:
pass
else:
return chains_in_month