def __call__(self, s):
self.result = None
def onStart(el):
self.result = el
def onEnd():
pass
def onElement(el):
self.result.addChild(el)
s = re.sub(r"\<\?xml.*\?\>\n", "", s, 1)
s = s.encode("ascii", "xmlcharrefreplace")
parser = domish.elementStream()
parser.DocumentStartEvent = onStart
parser.ElementEvent = onElement
parser.DocumentEndEvent = onEnd
tmp = domish.Element(("", "s"))
try:
tmp.addRawXml(s)
parser.parse(tmp.toXml())
except domish.ParserError:
return None
return self.result.firstChildElement()
def __init__(self):
# Setup the parser
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
self.config = None
def _reset(self):
# Setup the parser
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
self.entity = None
def __call__(self, s):
self.result = None
def onStart(el):
self.result = el
def onEnd():
pass
def onElement(el):
self.result.addChild(el)
s = re.sub(r"\<\?xml.*\?\>\n", '', s, 1)
s = s.encode('ascii', 'xmlcharrefreplace')
parser = domish.elementStream()
parser.DocumentStartEvent = onStart
parser.ElementEvent = onElement
parser.DocumentEndEvent = onEnd
tmp = domish.Element(("", "s"))
try:
tmp.addRawXml(s)
parser.parse(tmp.toXml())
except domish.ParserError:
return None
return self.result.firstChildElement()
def __init__(self):
# Setup the parser
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
self.hash = {}
self.files = {}
self.room = {}
def do(nwsli):
"""work"""
res = ""
xml = requests.get(
("https://water.weather.gov/ahps2/"
"hydrograph_to_xml.php?gage=%s&output=xml") % (nwsli, )).content
elementStream = domish.elementStream()
roots = []
results = []
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
res += """IEM %s AHPS2WXC host=0 TimeStamp=%s
5
15 Station
6 UTCDate
4 UTCTime
7 Stage
7 CFS\n""" % (
nwsli,
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
)
elementStream.parse(xml)
elem = results[0]
nodes = xpath.queryForNodes("/site/forecast/datum", elem)
if nodes is None:
return res
i = 0
maxval = {"val": 0, "time": None}
for node in nodes:
utc = datetime.datetime.strptime(
str(node.valid)[:15], "%Y-%m-%dT%H:%M")
res += ("%12s%03i %6s %4s %7s %7s\n") % (
nwsli,
i,
utc.strftime("%b %-d"),
utc.strftime("%H%M"),
node.primary,
node.secondary,
)
if float(str(node.primary)) > maxval["val"]:
maxval["val"] = float(str(node.primary))
maxval["time"] = utc
maxval["cfs"] = float(str(node.secondary))
i += 1
if maxval["time"] is not None:
utc = maxval["time"]
res += ("%12sMAX %6s %4s %7s %7s\n") % (
nwsli,
utc.strftime("%b %-d"),
utc.strftime("%H%M"),
maxval["val"],
maxval["cfs"],
)
return res
def __init__(self, src):
try:
stream = elementStream()
stream.ElementEvent = self.on_element
stream.DocumentStartEvent = self.onDocumentStart
stream.DocumentEndEvent = self.onDocumentEnd
stream.parse("<root>%s</root>" % (src))
except Exception, err:
logging.error("Error parsing XML from %s, %s", src, err)
def connectionMade(self):
# Setup the parser
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
self.dispatch(self, STREAM_CONNECTED_EVENT)
self.authenticator.connectionMade()
def connectionMade(self):
self.listeners = []
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onStanzaOrEvent
self.stream.DocumentEndEvent = self.onDocumentEnd
self.onConnect()
self.iq_count = 0
def connectionMade(self):
self.listeners = []
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onStanzaOrEvent
self.stream.DocumentEndEvent = self.onDocumentEnd
self.onConnect()
self.iq_count = 0
def _reset(self):
# need this to be in xmlstream
self.xmlstream.stream = domish.elementStream()
self.xmlstream.stream.DocumentStartEvent = self.xmlstream.onDocumentStart
self.xmlstream.stream.ElementEvent = self.xmlstream.onElement
self.xmlstream.stream.DocumentEndEvent = self.xmlstream.onDocumentEnd
self.xmlstream.prefixes = deepcopy(XMPP_PREFIXES)
# Generate stream header
if self.version != 0.0:
sh = "<stream:stream xmlns='%s' xmlns:stream='http://etherx.jabber.org/streams' version='%s' to='%s'>" % \
(self.namespace,self.version, self.streamHost.encode('utf-8'))
self.xmlstream.send(str(sh))
def _reset(self):
# need this to be in xmlstream
self.xmlstream.stream = domish.elementStream()
self.xmlstream.stream.DocumentStartEvent = self.xmlstream.onDocumentStart
self.xmlstream.stream.ElementEvent = self.xmlstream.onElement
self.xmlstream.stream.DocumentEndEvent = self.xmlstream.onDocumentEnd
self.xmlstream.prefixes = deepcopy(XMPP_PREFIXES)
# Generate stream header
if self.version != 0.0:
sh = "<stream:stream xmlns='%s' xmlns:stream='http://etherx.jabber.org/streams' version='%s' to='%s'>" % \
(self.namespace,self.version, self.streamHost.encode('utf-8'))
self.xmlstream.send(str(sh))
def _reset(self):
"""
Setup the parser
"""
if not self.use_t:
self.stream = elementStream()
else:
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
self.body = ""
self.xmpp_elements = []
def _reset(self):
"""
Setup the parser
"""
if not self.use_t:
self.stream = elementStream()
else:
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
self.body = ""
self.xmpp_elements = []
def parseXML(serialized):
result = []
def onStart(el):
result.append(el)
def onElement(el):
result[0].addChild(el)
parser = domish.elementStream()
parser.DocumentStartEvent = onStart
parser.ElementEvent = onElement
parser.DocumentEndEvent = lambda: None
parser.parse("<r>" + serialized + "</r>")
return result[0].children
def parseXML(serialized):
result = []
def onStart(el):
result.append(el)
def onElement(el):
result[0].addChild(el)
parser = domish.elementStream()
parser.DocumentStartEvent = onStart
parser.ElementEvent = onElement
parser.DocumentEndEvent = lambda: None
parser.parse("<r>" + serialized + "</r>")
return result[0].children
def __call__(self, xml):
self.result = None
def onStart(el):
self.result = el
def onEnd():
pass
def onElement(el):
self.result.addChild(el)
parser = domish.elementStream()
parser.DocumentStartEvent = onStart
parser.ElementEvent = onElement
parser.DocumentEndEvent = onEnd
parser.parse(xml)
return self.result
def parseXml(string):
"""
Parse serialized XML into a DOM structure.
@param string: The serialized XML to be parsed, UTF-8 encoded.
@type string: C{str}.
@return: The DOM structure, or C{None} on empty or incomplete input.
@rtype: L{domish.Element}
"""
roots = []
results = []
elementStream = domish.elementStream()
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
elementStream.parse(string)
return results and results[0] or None
def parseXml(string):
"""
Parse serialized XML into a DOM structure.
@param string: The serialized XML to be parsed, UTF-8 encoded.
@type string: C{str}.
@return: The DOM structure, or C{None} on empty or incomplete input.
@rtype: L{domish.Element}
"""
roots = []
results = []
elementStream = domish.elementStream()
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
elementStream.parse(string)
return results and results[0] or None
def process_site( nwsli, network ):
url = "http://water.weather.gov/ahps2/hydrograph_to_xml.php?gage=%s&output=xml" % (nwsli,)
elementStream = domish.elementStream()
roots = []
results = []
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
try:
xml = urllib2.urlopen(url).read()
except:
print "DOWNLOAD ERROR"
print url
return
try:
elementStream.parse(xml)
except:
print "XML ERROR"
print url
return
elem = results[0]
nodes = xpath.queryForNodes('/site/sigstages', elem)
sigstages = nodes[0]
data = {'id': nwsli, 'network': network}
for s in sigstages.children:
val = str(s)
if val == '':
val = None
data['sigstage_%s' %(s.name,)] = val
if not data.has_key('sigstage_low'):
print 'No Data', nwsli, network
return
print data
mcursor2.execute("""UPDATE stations SET sigstage_low = %(sigstage_low)s,
sigstage_action = %(sigstage_action)s, sigstage_bankfull = %(sigstage_bankfull)s,
sigstage_flood = %(sigstage_flood)s, sigstage_moderate = %(sigstage_moderate)s,
sigstage_major = %(sigstage_major)s, sigstage_record = %(sigstage_record)s
WHERE id = %(id)s and network = %(network)s """, data)
def do(nwsli):
res = ""
xml = urllib2.urlopen(('http://water.weather.gov/ahps2/'
'hydrograph_to_xml.php?gage=%s&output=xml'
) % (nwsli,)).read()
elementStream = domish.elementStream()
roots = []
results = []
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
res += """IEM %s AHPS2WXC host=0 TimeStamp=%s
5
15 Station
6 UTCDate
4 UTCTime
7 Stage
7 CFS\n""" % (nwsli,
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
)
elementStream.parse(xml)
elem = results[0]
nodes = xpath.queryForNodes('/site/forecast/datum', elem)
i = 0
maxval = {'val': 0, 'time': None}
for node in nodes:
utc = datetime.datetime.strptime(str(node.valid)[:15],
'%Y-%m-%dT%H:%M')
res += ("%12s%03i %6s %4s %7s %7s\n"
) % (nwsli, i, utc.strftime("%b %-d"), utc.strftime("%H%M"),
node.primary, node.secondary)
if float(str(node.primary)) > maxval['val']:
maxval['val'] = float(str(node.primary))
maxval['time'] = utc
maxval['cfs'] = float(str(node.secondary))
i += 1
if maxval['time'] is not None:
utc = maxval['time']
res += ("%12sMAX %6s %4s %7s %7s\n"
) % (nwsli, utc.strftime("%b %-d"), utc.strftime("%H%M"),
maxval['val'], maxval['cfs'])
return res
def do(nwsli):
res = ""
xml = urllib2.urlopen(('http://water.weather.gov/ahps2/'
'hydrograph_to_xml.php?gage=%s&output=xml'
) % (nwsli,)).read()
elementStream = domish.elementStream()
roots = []
results = []
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
res += """IEM %s AHPS2WXC host=0 TimeStamp=%s
5
15 Station
6 UTCDate
4 UTCTime
7 Stage
7 CFS\n""" % (nwsli,
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
)
elementStream.parse(xml)
elem = results[0]
nodes = xpath.queryForNodes('/site/forecast/datum', elem)
i = 0
maxval = {'val': 0, 'time': None}
for node in nodes:
utc = datetime.datetime.strptime(str(node.valid)[:15],
'%Y-%m-%dT%H:%M')
res += ("%12s%03i %6s %4s %7s %7s\n"
) % (nwsli, i, utc.strftime("%b %-d"), utc.strftime("%H%M"),
node.primary, node.secondary)
if float(str(node.primary)) > maxval['val']:
maxval['val'] = float(str(node.primary))
maxval['time'] = utc
maxval['cfs'] = float(str(node.secondary))
i += 1
if maxval['time'] is not None:
utc = maxval['time']
res += ("%12sMAX %6s %4s %7s %7s\n"
) % (nwsli, utc.strftime("%b %-d"), utc.strftime("%H%M"),
maxval['val'], maxval['cfs'])
return res
def main():
elements = []
parser = domish.elementStream()
parser.DocumentStartEvent = lambda e: None
parser.DocumentEndEvent = lambda: None
parser.ElementEvent = elements.append
parser.parse(testDocument)
count = 1000
before_py = time.time()
slowfunc_py(elements, count)
after_py = time.time()
before_c = time.time()
slowfunc_c(elements, count)
after_c = time.time()
print 'py: Serialized %d elements in %0.2f seconds - %d elements/second' % (
count * len(elements),
after_py - before_py,
(count * len(elements)) / (after_py - before_py))
print ' c: Serialized %d elements in %0.2f seconds - %d elements/second' % (
count * len(elements),
after_c - before_c,
(count * len(elements)) / (after_c - before_c))
def __call__(self, s):
self.result = None
def onStart(el):
self.result = el
def onEnd():
pass
def onElement(el):
self.result.addChild(el)
parser = domish.elementStream()
parser.DocumentStartEvent = onStart
parser.ElementEvent = onElement
parser.DocumentEndEvent = onEnd
tmp = domish.Element(("", "s"))
tmp.addRawXml(s)
parser.parse(tmp.toXml())
return self.result.firstChildElement()
def _initializeStream(self):
""" Sets up XML Parser. """
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
def process_site(mcursor, nwsli, network):
"""Do our processing work"""
url = ("http://water.weather.gov/ahps2/hydrograph_to_xml.php?"
"gage=%s&output=xml") % (nwsli, )
elementStream = domish.elementStream()
roots = []
results = []
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
try:
req = requests.get(url, timeout=30)
xml = req.content
if xml.strip() == "No results found for this gage.":
print("No results for %s" % (nwsli, ))
return
except Exception as exp:
print("DOWNLOAD ERROR")
print(url)
print(exp)
return
try:
elementStream.parse(xml)
except Exception as exp:
print("XML ERROR")
print(url)
print(exp)
return
elem = results[0]
nodes = xpath.queryForNodes("/site/sigstages", elem)
if nodes is None:
print("No data found for %s" % (nwsli, ))
return
sigstages = nodes[0]
data = {
"id": nwsli,
"network": network,
"sigstage_low": None,
"sigstage_action": None,
"sigstage_bankfull": None,
"sigstage_flood": None,
"sigstage_moderate": None,
"sigstage_major": None,
"sigstage_record": None,
}
for s in sigstages.children:
val = str(s)
if val == "":
continue
data["sigstage_%s" % (s.name, )] = float(val)
if "sigstage_low" not in data:
print("No Data %s %s" % (nwsli, network))
return
print(("%s %5.1f %5.1f %5.1f %5.1f %5.1f %5.1f %5.1f") % (
data["id"],
data["sigstage_low"] or -99,
data["sigstage_action"] or -99,
data["sigstage_bankfull"] or -99,
data["sigstage_flood"] or -99,
data["sigstage_moderate"] or -99,
data["sigstage_major"] or -99,
data["sigstage_record"] or -99,
))
mcursor.execute(
"""
UPDATE stations SET sigstage_low = %(sigstage_low)s,
sigstage_action = %(sigstage_action)s,
sigstage_bankfull = %(sigstage_bankfull)s,
sigstage_flood = %(sigstage_flood)s,
sigstage_moderate = %(sigstage_moderate)s,
sigstage_major = %(sigstage_major)s,
sigstage_record = %(sigstage_record)s
WHERE id = %(id)s and network = %(network)s
""",
data,
)
def __init__(self):
self.elementStream = domish.elementStream()
self.elementStream.DocumentStartEvent = self.docStart
self.elementStream.ElementEvent = self.elem
self.elementStream.DocumentEndEvent = self.docEnd
self.done = False
def process_site(nwsli, network):
url = ("http://water.weather.gov/ahps2/hydrograph_to_xml.php?"
"gage=%s&output=xml") % (nwsli, )
elementStream = domish.elementStream()
roots = []
results = []
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
try:
xml = urllib2.urlopen(url).read()
if xml.strip() == 'No results found for this gage.':
print 'No results for %s' % (nwsli, )
return
except:
print "DOWNLOAD ERROR"
print url
return
try:
elementStream.parse(xml)
except:
print "XML ERROR"
print url
return
elem = results[0]
nodes = xpath.queryForNodes('/site/sigstages', elem)
if nodes is None:
print "No data found for", nwsli
return
sigstages = nodes[0]
data = {
'id': nwsli,
'network': network,
'sigstage_low': None,
'sigstage_action': None,
'sigstage_bankfull': None,
'sigstage_flood': None,
'sigstage_moderate': None,
'sigstage_major': None,
'sigstage_record': None
}
for s in sigstages.children:
val = str(s)
if val == '':
continue
data['sigstage_%s' % (s.name, )] = float(val)
if 'sigstage_low' not in data:
print 'No Data', nwsli, network
return
print(("%s %5.1f %5.1f %5.1f %5.1f %5.1f %5.1f %5.1f") %
(data['id'], data['sigstage_low'] or -99, data['sigstage_action']
or -99, data['sigstage_bankfull'] or -99, data['sigstage_flood']
or -99, data['sigstage_moderate'] or -99, data['sigstage_major']
or -99, data['sigstage_record'] or -99))
mcursor.execute(
"""UPDATE stations SET sigstage_low = %(sigstage_low)s,
sigstage_action = %(sigstage_action)s,
sigstage_bankfull = %(sigstage_bankfull)s,
sigstage_flood = %(sigstage_flood)s,
sigstage_moderate = %(sigstage_moderate)s,
sigstage_major = %(sigstage_major)s, sigstage_record = %(sigstage_record)s
WHERE id = %(id)s and network = %(network)s """, data)
def _initializeStream(self):
""" Sets up XML Parser. """
self.stream = domish.elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
def validateXml(xml):
stream = elementStream()
stream.DocumentStartEvent = lambda e: None
stream.ElementEvent = lambda e: None
stream.DocumentEndEvent = lambda: None
stream.parse(xml) #will throw a ParserError if xml is not well-formed
def process_site(nwsli, network):
url = ("http://water.weather.gov/ahps2/hydrograph_to_xml.php?"
"gage=%s&output=xml") % (nwsli,)
elementStream = domish.elementStream()
roots = []
results = []
elementStream.DocumentStartEvent = roots.append
elementStream.ElementEvent = lambda elem: roots[0].addChild(elem)
elementStream.DocumentEndEvent = lambda: results.append(roots[0])
try:
xml = urllib2.urlopen(url).read()
if xml.strip() == 'No results found for this gage.':
print 'No results for %s' % (nwsli,)
return
except:
print "DOWNLOAD ERROR"
print url
return
try:
elementStream.parse(xml)
except:
print "XML ERROR"
print url
return
elem = results[0]
nodes = xpath.queryForNodes('/site/sigstages', elem)
if nodes is None:
print "No data found for", nwsli
return
sigstages = nodes[0]
data = {'id': nwsli, 'network': network, 'sigstage_low': None,
'sigstage_action': None, 'sigstage_bankfull': None,
'sigstage_flood': None, 'sigstage_moderate': None,
'sigstage_major': None, 'sigstage_record': None}
for s in sigstages.children:
val = str(s)
if val == '':
continue
data['sigstage_%s' % (s.name, )] = float(val)
if 'sigstage_low' not in data:
print 'No Data', nwsli, network
return
print(("%s %5.1f %5.1f %5.1f %5.1f %5.1f %5.1f %5.1f"
) % (data['id'], data['sigstage_low'] or -99,
data['sigstage_action'] or -99,
data['sigstage_bankfull'] or -99,
data['sigstage_flood'] or -99,
data['sigstage_moderate'] or -99,
data['sigstage_major'] or -99,
data['sigstage_record'] or -99))
mcursor.execute("""UPDATE stations SET sigstage_low = %(sigstage_low)s,
sigstage_action = %(sigstage_action)s,
sigstage_bankfull = %(sigstage_bankfull)s,
sigstage_flood = %(sigstage_flood)s,
sigstage_moderate = %(sigstage_moderate)s,
sigstage_major = %(sigstage_major)s, sigstage_record = %(sigstage_record)s
WHERE id = %(id)s and network = %(network)s """, data)
def _reset(self):
self.stream = elementStream()
self.stream.DocumentStartEvent = self.onDocumentStart
self.stream.ElementEvent = self.onElement
self.stream.DocumentEndEvent = self.onDocumentEnd
def __init__(self):
self.elementStream = domish.elementStream()
self.elementStream.DocumentStartEvent = self.docStart
self.elementStream.ElementEvent = self.elem
self.elementStream.DocumentEndEvent = self.docEnd
self.done = False
def validateXml(xml):
stream = elementStream()
stream.DocumentStartEvent = lambda e: None
stream.ElementEvent = lambda e: None
stream.DocumentEndEvent = lambda: None
stream.parse(xml) #will throw a ParserError if xml is not well-formed
