def _xml_to_log_message(self, output):
"""
author - string - the name of the author who committed the revision
date - float time - the date of the commit
message - string - the text of the log message for the commit
revision - pysvn.Revision - the revision of the commit
changed_paths - list of dictionaries. Each dictionary contains:
path - string - the path in the repository
action - string
copyfrom_path - string - if copied, the original path, else None
copyfrom_revision - pysvn.Revision - if copied, the revision of the original, else None
"""
doc = ET.fromstring(output)
entries = []
for logentry in doc:
entry = dict(
revision = CommandLineRevision(logentry.get('revision')),
author = logentry.findtext("author"),
date = self._xmldate_to_timestamp(logentry.findtext("date")),
message = logentry.findtext("msg"),
)
entry['changed_paths'] = []
paths = logentry.find("paths")
if paths:
for path in paths:
cp = CommandLineChangedPath()
cp.kind = path.get('kind')
cp.action = path.get('action')
cp.path = path.text
entry['changed_paths'].append(cp)
entries.append(entry)
return entries
def remote_get(self, id):
url = urljoin(
self.api_url,
'view/%s?%s' % (id.isalnum() and id + '/nocomment' or quote(id),
urlencode({
'language': self.fml_lang,
'key': self.api_key
})))
f = urlopen(url)
try:
tree = ElementTree.parse(f)
except SyntaxError:
class_, e, tb = exc_info()
new_exc = FMLException(u'XML Parsing Error: %s' % unicode(e))
raise new_exc.__class__, new_exc, tb
if tree.find('.//error'):
raise FMLException(tree.findtext('.//error'))
item = tree.find('.//item')
if item:
url = u"http://www.fmylife.com/%s/%s" % (
item.findtext('category'),
item.get('id'),
)
text = item.find('text').text
return u'%s\n- %s' % (text, url)
def _xml_to_log_message(self, output):
"""
author - string - the name of the author who committed the revision
date - float time - the date of the commit
message - string - the text of the log message for the commit
revision - pysvn.Revision - the revision of the commit
changed_paths - list of dictionaries. Each dictionary contains:
path - string - the path in the repository
action - string
copyfrom_path - string - if copied, the original path, else None
copyfrom_revision - pysvn.Revision - if copied, the revision of the original, else None
"""
doc = ET.fromstring(output)
entries = []
for logentry in doc:
entry = dict(
revision=CommandLineRevision(logentry.get('revision')),
author=logentry.findtext("author"),
date=self._xmldate_to_timestamp(logentry.findtext("date")),
message=logentry.findtext("msg"),
)
entry['changed_paths'] = []
paths = logentry.find("paths")
if paths:
for path in paths:
cp = CommandLineChangedPath()
cp.kind = path.get('kind')
cp.action = path.get('action')
cp.path = path.text
entry['changed_paths'].append(cp)
entries.append(entry)
return entries
def calc(self, event, expression):
tree = self._google_scrape_search(expression)
nodes = [node for node in tree.findall(".//h2/b")]
if len(nodes) == 1:
# ElementTree doesn't support inline tags:
# May return ASCII unless an encoding is specified.
# "utf8" will result in an xml header
node = ElementTree.tostring(nodes[0], encoding="utf-8")
node = node.decode("utf-8")
node = re.sub(r"<sup>(.*?)</sup>", lambda x: u"^" + x.group(1), node)
node = re.sub(r"<.*?>", "", node)
node = re.sub(r"(\d)\s+(\d)", lambda x: x.group(1) + x.group(2), node)
node = decode_htmlentities(node)
event.addresponse(node)
else:
event.addresponse(u"%s, Google wasn't interested in calculating that", choice(("Sorry", "Whoops")))
def calc(self, event, expression):
tree = self._google_scrape_search(expression)
nodes = [node for node in tree.findall('.//h2/b')]
if len(nodes) == 1:
# ElementTree doesn't support inline tags:
# May return ASCII unless an encoding is specified.
# "utf8" will result in an xml header
node = ElementTree.tostring(nodes[0], encoding='utf-8')
node = node.decode('utf-8')
node = re.sub(r'<sup>(.*?)</sup>',
lambda x: u'^' + x.group(1), node)
node = re.sub(r'<.*?>', '', node)
node = re.sub(r'(\d)\s+(\d)', lambda x: x.group(1) + x.group(2),
node)
node = decode_htmlentities(node)
event.addresponse(node)
else:
event.addresponse(u'No result')
def bible(self, event, passage, version=None):
psalm_pat = re.compile(r'\bpsalm\b', re.IGNORECASE)
passage = psalm_pat.sub('psalms', passage)
params = {'passage': passage.encode('utf-8'),
'type': 'xml',
'formatting': 'plain'}
if version:
params['version'] = version.lower().encode('utf-8')
f = urlopen(self.api_url + '?' + urlencode(params))
tree = ElementTree.parse(f)
message = self.formatPassage(tree)
if message:
event.addresponse(message)
errors = list(tree.findall('.//error'))
if errors:
event.addresponse(u'There were errors: %s.', '. '.join(err.text for err in errors))
elif not message:
event.addresponse(u"I couldn't find that passage.")
def remote_latest(self, service, user):
if service['api'] == 'twitter':
# Twitter ommits retweets in the JSON and XML results:
statuses = generic_webservice(
'%sstatuses/user_timeline/%s.atom' %
(service['endpoint'], user.encode('utf-8')), {'count': 1})
tree = ElementTree.fromstring(statuses)
latest = tree.find('{http://www.w3.org/2005/Atom}entry')
if latest is None:
raise self.NoTweetsException(user)
return {
'text':
latest.findtext('{http://www.w3.org/2005/Atom}content').split(
': ', 1)[1],
'ago':
ago(datetime.utcnow() - parse_timestamp(
latest.findtext('{http://www.w3.org/2005/Atom}published'))
),
'url': [
x for x in latest.getiterator(
'{http://www.w3.org/2005/Atom}link')
if x.get('type') == 'text/html'
][0].get('href'),
}
elif service['api'] == 'laconica':
statuses = json_webservice(
'%sstatuses/user_timeline/%s.json' %
(service['endpoint'], user.encode('utf-8')), {'count': 1})
if not statuses:
raise self.NoTweetsException(user)
latest = statuses[0]
url = '%s/notice/%i' % (service['endpoint'].split(
'/api/', 1)[0], latest['id'])
return {
'text': decode_htmlentities(latest['text']),
'ago':
ago(datetime.utcnow() - parse_timestamp(latest['created_at'])),
'url': url,
}
def remote_latest(self, service, user):
if service['api'] == 'twitter':
# Twitter ommits retweets in the JSON and XML results:
statuses = generic_webservice(
'%sstatuses/user_timeline.rss' % service['endpoint'], {
'screen_name': user,
'count': 1
})
tree = ElementTree.fromstring(statuses)
latest = tree.find('.//item')
if latest is None:
if tree.find('.//error'):
log.info('Twitter user_latest returned: %s',
tree.findtext('.//error'))
raise self.NoTweetsException(user)
return {
'text':
latest.findtext('description').split(': ', 1)[1],
'ago':
ago(datetime.utcnow() -
parse_timestamp(latest.findtext('pubDate'))),
'url':
latest.findtext('guid'),
}
elif service['api'] == 'laconica':
statuses = json_webservice(
'%sstatuses/user_timeline/%s.json' %
(service['endpoint'], user.encode('utf-8')), {'count': 1})
if not statuses:
raise self.NoTweetsException(user)
latest = statuses[0]
url = '%s/notice/%i' % (service['endpoint'].split(
'/api/', 1)[0], latest['id'])
return {
'text': decode_htmlentities(latest['text']),
'ago':
ago(datetime.utcnow() - parse_timestamp(latest['created_at'])),
'url': url,
}
def remote_latest(self, service, user):
if service['api'] == 'twitter':
# Twitter ommits retweets in the JSON and XML results:
statuses = generic_webservice('%sstatuses/user_timeline.rss'
% service['endpoint'], {
'screen_name': user,
'count': 1
})
tree = ElementTree.fromstring(statuses)
latest = tree.find('.//item')
if latest is None:
if tree.find('.//error'):
log.info('Twitter user_latest returned: %s',
tree.findtext('.//error'))
raise self.NoTweetsException(user)
return {
'text': latest.findtext('description')
.split(': ', 1)[1],
'ago': ago(datetime.utcnow() - parse_timestamp(
latest.findtext('pubDate'))),
'url': latest.findtext('guid'),
}
elif service['api'] == 'laconica':
statuses = json_webservice('%sstatuses/user_timeline/%s.json'
% (service['endpoint'], user.encode('utf-8')),
{'count': 1})
if not statuses:
raise self.NoTweetsException(user)
latest = statuses[0]
url = '%s/notice/%i' % (service['endpoint'].split('/api/', 1)[0],
latest['id'])
return {
'text': decode_htmlentities(latest['text']),
'ago': ago(datetime.utcnow()
- parse_timestamp(latest['created_at'])),
'url': url,
}
def bible(self, event, passage, version=None):
passage = self.psalm_pat.sub('psalms', passage)
params = {
'passage': passage.encode('utf-8'),
'type': 'xml',
'formatting': 'plain'
}
if version:
params['version'] = version.lower().encode('utf-8')
f = urlopen(self.api_url + '?' + urlencode(params))
tree = ElementTree.parse(f)
message = self.formatPassage(tree)
if message:
event.addresponse(message)
errors = list(tree.findall('.//error'))
if errors:
event.addresponse(u'There were errors: %s.',
'. '.join(err.text for err in errors))
elif not message:
event.addresponse(u"I couldn't find that passage.")
def remote_get(self, id):
url = urljoin(self.api_url, 'view/%s?%s' % (
id.isalnum() and id + '/nocomment' or quote(id),
urlencode({'language': self.fml_lang, 'key': self.api_key}))
)
f = urlopen(url)
try:
tree = ElementTree.parse(f)
except SyntaxError:
class_, e, tb = exc_info()
new_exc = FMLException(u'XML Parsing Error: %s' % unicode(e))
raise new_exc.__class__, new_exc, tb
if tree.find('.//error'):
raise FMLException(tree.findtext('.//error'))
item = tree.find('.//item')
if item:
url = u"http://www.fmylife.com/%s/%s" % (
item.findtext('category'),
item.get('id'),
)
text = item.find('text').text
return u'%s\n- %s' % (text, url)
def remote_latest(self, service, user):
if service['api'] == 'twitter':
# Twitter ommits retweets in the JSON and XML results:
statuses = generic_webservice('%sstatuses/user_timeline/%s.atom'
% (service['endpoint'], user.encode('utf-8')),
{'count': 1})
tree = ElementTree.fromstring(statuses)
latest = tree.find('{http://www.w3.org/2005/Atom}entry')
if latest is None:
raise self.NoTweetsException(user)
return {
'text': latest.findtext('{http://www.w3.org/2005/Atom}content')
.split(': ', 1)[1],
'ago': ago(datetime.utcnow() - parse_timestamp(
latest.findtext('{http://www.w3.org/2005/Atom}published'))),
'url': [x for x
in latest.getiterator('{http://www.w3.org/2005/Atom}link')
if x.get('type') == 'text/html'
][0].get('href'),
}
elif service['api'] == 'laconica':
statuses = json_webservice('%sstatuses/user_timeline/%s.json'
% (service['endpoint'], user.encode('utf-8')),
{'count': 1})
if not statuses:
raise self.NoTweetsException(user)
latest = statuses[0]
url = '%s/notice/%i' % (service['endpoint'].split('/api/', 1)[0],
latest['id'])
return {
'text': decode_htmlentities(latest['text']),
'ago': ago(datetime.utcnow()
- parse_timestamp(latest['created_at'])),
'url': url,
}
def calc(self, event, expression):
tree = self._google_scrape_search(expression)
nodes = [
node for node in tree.findall('.//h2') if node.get('class') == 'r'
]
if len(nodes) == 1:
# ElementTree doesn't support inline tags:
# May return ASCII unless an encoding is specified.
# "utf8" will result in an xml header
node = ElementTree.tostring(nodes[0], encoding='utf-8')
node = node.decode('utf-8')
node = re.sub(r'<sup>(.*?)</sup>', lambda x: u'^' + x.group(1),
node)
node = re.sub(r'<.*?>', '', node)
node = re.sub(r'(\d)\s+(\d)', lambda x: x.group(1) + x.group(2),
node)
node = decode_htmlentities(node)
node = re.sub(r'\s+', ' ', node)
event.addresponse(node)
else:
event.addresponse(
u"%s, Google wasn't interested in calculating that",
choice(('Sorry', 'Whoops')))
def _load_currencies(self):
iso4127_file = cacheable_download(
'http://www.currency-iso.org/dl_iso_table_a1.xml',
'conversions/iso4217.xml')
document = ElementTree.parse(iso4127_file)
# Code -> [Countries..., Currency Name, post-decimal digits]
self.currencies = {}
# Country -> Code
self.country_currencies = {}
self.country_codes = get_country_codes()
# Non-currencies:
non_currencies = set((
'BOV CLF COU MXV '
'UYI XSU XUA ' # Various Fund codes
'CHE CHW ' # Swiss WIR currencies
'USN USS ' # US Dollar fund codes
'XAG XAU XPD XPT ' # Metals
'XBA XBB XBC XBD ' # Euro Bond Market
'XDR XTS XXX ' # Other specials
).split())
no_country_codes = set((
'Saint Martin',
'Virgin Islands (Us)',
'Virgin Islands (British)',
))
accociated_all_countries = True
for currency in document.getiterator('ISO_CURRENCY'):
code = currency.findtext('ALPHABETIC_CODE').strip()
name = currency.findtext('CURRENCY').strip()
place = currency.findtext('ENTITY').strip().title()
try:
minor_units = int(currency.findtext('MINOR_UNIT').strip())
except ValueError:
minor_units = 2
if code == '' or code in non_currencies:
continue
# Fund codes
if re.match(r'^Zz[0-9]{2}', place, re.UNICODE):
continue
if code in self.currencies:
self.currencies[code][0].append(place)
else:
self.currencies[code] = [[place], name, minor_units]
if place in no_country_codes:
continue
if (code[:2] in self.country_codes
and code[:2] not in self.country_currencies):
self.country_currencies[code[:2]] = code
continue
ascii_place = (unicodedata.normalize('NFD', unicode(place)).encode(
'ASCII', 'ignore').replace('-', ' ').replace('Sint', 'Saint'))
# Countries with (alternative names)
swapped_place = None
m = re.match(r'^(.+?)\s+\((.+)\)$', ascii_place)
if m is not None:
swapped_place = '%s (%s)' % (m.group(2), m.group(1))
for ccode, country in self.country_codes.iteritems():
country = country.title()
ascii_country = (unicodedata.normalize('NFD', country).encode(
'ASCII', 'ignore').replace('-',
' ').replace('Sint', 'Saint'))
if ascii_country in (ascii_place, swapped_place):
if ccode not in self.country_currencies:
self.country_currencies[ccode] = code
break
else:
log.info(
u"ISO4127 parsing: Can't identify %s as a known "
u"country", place)
accociated_all_countries = False
# Special cases for shared currencies:
self.currencies['EUR'][0].append(u'Euro Member Countries')
self.currencies['XAF'][0].append(
u"Communaut\xe9 financi\xe8re d'Afrique")
self.currencies['XCD'][0].append(
u'Organisation of Eastern Caribbean States')
self.currencies['XOF'][0].append(
u'Coop\xe9ration financi\xe8re en Afrique centrale')
self.currencies['XPF'][0].append(u'Comptoirs Fran\xe7ais du Pacifique')
return accociated_all_countries
def _load_currencies(self):
iso4127_file = cacheable_download(
'http://www.currency-iso.org/dam/isocy/downloads'
'/dl_iso_table_a1.xml',
'conversions/iso4217.xml')
document = ElementTree.parse(iso4127_file)
# Code -> [Countries..., Currency Name, post-decimal digits]
self.currencies = {}
# Country -> Code
self.country_currencies = {}
self.country_codes = get_country_codes()
# Non-currencies:
non_currencies = set(('BOV CLF COU MXV '
'UYI XSU XUA ' # Various Fund codes
'CHE CHW ' # Swiss WIR currencies
'USN USS ' # US Dollar fund codes
'XAG XAU XPD XPT ' # Metals
'XBA XBB XBC XBD ' # Euro Bond Market
'XDR XTS XXX ' # Other specials
).split())
no_country_codes = set(('Saint Martin',
'Virgin Islands (Us)',
'Virgin Islands (British)',))
accociated_all_countries = True
for currency in document.getiterator('ISO_CURRENCY'):
code = currency.findtext('ALPHABETIC_CODE').strip()
name = currency.findtext('CURRENCY').strip()
place = currency.findtext('ENTITY').strip().title()
try:
minor_units = int(currency.findtext('MINOR_UNIT').strip())
except ValueError:
minor_units = 2
if code == '' or code in non_currencies:
continue
# Fund codes
if re.match(r'^Zz[0-9]{2}', place, re.UNICODE):
continue
if code in self.currencies:
self.currencies[code][0].append(place)
else:
self.currencies[code] = [[place], name, minor_units]
if place in no_country_codes:
continue
if (code[:2] in self.country_codes
and code[:2] not in self.country_currencies):
self.country_currencies[code[:2]] = code
continue
# Countries with (alternative names)
swapped_place = None
m = re.match(r'^(.+?)\s+\((.+)\)$', place)
if m is not None:
swapped_place = '%s (%s)' % (m.group(2), m.group(1))
for ccode, country in self.country_codes.iteritems():
country = country.title()
if country in (place, swapped_place):
if ccode not in self.country_currencies:
self.country_currencies[ccode] = code
break
else:
log.info(u"ISO4127 parsing: Can't identify %s as a known "
u"country", place)
accociated_all_countries = False
# Special cases for shared currencies:
self.currencies['EUR'][0].append(u'Euro Member Countries')
self.currencies['XAF'][0].append(u"Communaut\xe9 financi\xe8re d'Afrique")
self.currencies['XCD'][0].append(u'Organisation of Eastern Caribbean States')
self.currencies['XOF'][0].append(u'Coop\xe9ration financi\xe8re en Afrique centrale')
self.currencies['XPF'][0].append(u'Comptoirs Fran\xe7ais du Pacifique')
return accociated_all_countries
