def read_airport_data(self): # File is listed as ISO 8859-1 (Latin-1) encoded on # http://openflights.org/data.html, but from decoding it appears to # actually be UTF8 filename = cacheable_download(self.airports_url, u'flight/airports.dat') reader = csv.reader(open(filename), delimiter=',', quotechar='"') for row in reader: self.airports[int(row[0])] = [unicode(r, u'utf-8') for r in row[1:]]
def update(self, max_age=None): headers = {} if max_age: headers['Cache-Control'] = 'max-age=%i' % max_age feedfile = cacheable_download(self.url, "feeds/%s-%i.xml" % ( re.sub(r'\W+', '_', self.name), self.identity_id), headers) self.feed = feedparser.parse(feedfile) self.entries = self.feed['entries']
def lookup_mac(self, event, _, mac): oui = mac.replace('-', '').replace(':', '').upper()[:6] ouis = open(cacheable_download('http://standards.ieee.org/regauth/oui/oui.txt', 'sysadmin/oui.txt')) match = re.search(r'^\s*%s\s+\(base 16\)\s+(.+?)$' % oui, ouis.read(), re.MULTILINE) ouis.close() if match: name = match.group(1).decode('utf8').title() event.addresponse(u"That belongs to %s", name) else: event.addresponse(u"I don't know who that belongs to")
def _load_currencies(self): iso4127_file = cacheable_download( 'http://www.currency-iso.org/dl_iso_table_a1.xml', 'conversions/iso4217.xml') document = ElementTree.parse(iso4127_file) # Code -> [Countries..., Currency Name, post-decimal digits] self.currencies = {} # Country -> Code self.country_currencies = {} self.country_codes = get_country_codes() # Non-currencies: non_currencies = set(( 'BOV CLF COU MXV ' 'UYI XSU XUA ' # Various Fund codes 'CHE CHW ' # Swiss WIR currencies 'USN USS ' # US Dollar fund codes 'XAG XAU XPD XPT ' # Metals 'XBA XBB XBC XBD ' # Euro Bond Market 'XDR XTS XXX ' # Other specials ).split()) no_country_codes = set(( 'Saint Martin', 'Virgin Islands (Us)', 'Virgin Islands (British)', )) accociated_all_countries = True for currency in document.getiterator('ISO_CURRENCY'): code = currency.findtext('ALPHABETIC_CODE').strip() name = currency.findtext('CURRENCY').strip() place = currency.findtext('ENTITY').strip().title() try: minor_units = int(currency.findtext('MINOR_UNIT').strip()) except ValueError: minor_units = 2 if code == '' or code in non_currencies: continue # Fund codes if re.match(r'^Zz[0-9]{2}', place, re.UNICODE): continue if code in self.currencies: self.currencies[code][0].append(place) else: self.currencies[code] = [[place], name, minor_units] if place in no_country_codes: continue if (code[:2] in self.country_codes and code[:2] not in self.country_currencies): self.country_currencies[code[:2]] = code continue ascii_place = (unicodedata.normalize('NFD', unicode(place)).encode( 'ASCII', 'ignore').replace('-', ' ').replace('Sint', 'Saint')) # Countries with (alternative names) swapped_place = None m = re.match(r'^(.+?)\s+\((.+)\)$', ascii_place) if m is not None: swapped_place = '%s (%s)' % (m.group(2), m.group(1)) for ccode, country in self.country_codes.iteritems(): country = country.title() ascii_country = (unicodedata.normalize('NFD', country).encode( 'ASCII', 'ignore').replace('-', ' ').replace('Sint', 'Saint')) if ascii_country in (ascii_place, swapped_place): if ccode not in self.country_currencies: self.country_currencies[ccode] = code break else: log.info( u"ISO4127 parsing: Can't identify %s as a known " u"country", place) accociated_all_countries = False # Special cases for shared currencies: self.currencies['EUR'][0].append(u'Euro Member Countries') self.currencies['XAF'][0].append( u"Communaut\xe9 financi\xe8re d'Afrique") self.currencies['XCD'][0].append( u'Organisation of Eastern Caribbean States') self.currencies['XOF'][0].append( u'Coop\xe9ration financi\xe8re en Afrique centrale') self.currencies['XPF'][0].append(u'Comptoirs Fran\xe7ais du Pacifique') return accociated_all_countries
def _load_currencies(self): iso4127_file = cacheable_download( 'http://www.currency-iso.org/dam/isocy/downloads' '/dl_iso_table_a1.xml', 'conversions/iso4217.xml') document = ElementTree.parse(iso4127_file) # Code -> [Countries..., Currency Name, post-decimal digits] self.currencies = {} # Country -> Code self.country_currencies = {} self.country_codes = get_country_codes() # Non-currencies: non_currencies = set(('BOV CLF COU MXV ' 'UYI XSU XUA ' # Various Fund codes 'CHE CHW ' # Swiss WIR currencies 'USN USS ' # US Dollar fund codes 'XAG XAU XPD XPT ' # Metals 'XBA XBB XBC XBD ' # Euro Bond Market 'XDR XTS XXX ' # Other specials ).split()) no_country_codes = set(('Saint Martin', 'Virgin Islands (Us)', 'Virgin Islands (British)',)) accociated_all_countries = True for currency in document.getiterator('ISO_CURRENCY'): code = currency.findtext('ALPHABETIC_CODE').strip() name = currency.findtext('CURRENCY').strip() place = currency.findtext('ENTITY').strip().title() try: minor_units = int(currency.findtext('MINOR_UNIT').strip()) except ValueError: minor_units = 2 if code == '' or code in non_currencies: continue # Fund codes if re.match(r'^Zz[0-9]{2}', place, re.UNICODE): continue if code in self.currencies: self.currencies[code][0].append(place) else: self.currencies[code] = [[place], name, minor_units] if place in no_country_codes: continue if (code[:2] in self.country_codes and code[:2] not in self.country_currencies): self.country_currencies[code[:2]] = code continue # Countries with (alternative names) swapped_place = None m = re.match(r'^(.+?)\s+\((.+)\)$', place) if m is not None: swapped_place = '%s (%s)' % (m.group(2), m.group(1)) for ccode, country in self.country_codes.iteritems(): country = country.title() if country in (place, swapped_place): if ccode not in self.country_currencies: self.country_currencies[ccode] = code break else: log.info(u"ISO4127 parsing: Can't identify %s as a known " u"country", place) accociated_all_countries = False # Special cases for shared currencies: self.currencies['EUR'][0].append(u'Euro Member Countries') self.currencies['XAF'][0].append(u"Communaut\xe9 financi\xe8re d'Afrique") self.currencies['XCD'][0].append(u'Organisation of Eastern Caribbean States') self.currencies['XOF'][0].append(u'Coop\xe9ration financi\xe8re en Afrique centrale') self.currencies['XPF'][0].append(u'Comptoirs Fran\xe7ais du Pacifique') return accociated_all_countries
event.addresponse(u'Done') @match(r'^usaco\s+(\S+)\s+results(?:\s+for\s+(.+))?$') def usaco_results(self, event, contest, user): if user is not None: try: usaco_user = self._get_usaco_user(event, user) except UsacoException, e: if 'down' in e.msg: event.addresponse(e) return usaco_user = user url = u'http://ace.delos.com/%sresults' % contest.upper() try: filename = cacheable_download(url, u'usaco/results_%s' % contest.upper(), timeout=30) except HTTPError: event.addresponse(u"Sorry, the results for %s aren't released yet", contest) except URLError: event.addresponse(u"Sorry, I couldn't fetch the USACO results. Maybe USACO is down?") if user is not None: users = {usaco_user: user.lower()} else: try: users = self._get_usaco_users(event) print users except UsacoException, e: event.addresponse(e) return
def _update_list(self): if not self.indexfile or time.time( ) - self.last_checked > self.cachetime: self.indexfile = cacheable_download(self.indexurl, "rfc/rfc-index.txt") self.last_checked = time.time()
def _update_list(self): if not self.indexfile or time.time() - self.last_checked > self.cachetime: self.indexfile = cacheable_download(self.indexurl, "rfc/rfc-index.txt") self.last_checked = time.time()