def process(self, ctx, m):
data = ctx.interpolate(self.data, m)
m[self.prefix + "country_code"] = self._gi.country_code_by_addr(data)
m[self.prefix + "country_name"] = self._gi.country_name_by_addr(
data) #.decode("latin1")
m[self.prefix + "cont_name"] = None
if m[self.prefix + "country_code"]:
try:
m[self.prefix + "cont_name"] = transformations.ccn_to_ctn(
transformations.cca2_to_ccn(
m[self.prefix + "country_code"])) #.decode("latin1")
except Exception as e:
if not self._extract_error and not ctx.debug2:
self._extract_error = True
logger.warning(
"Could not extract continent name from country code '%s' (reported only once per run) Error: %s"
% (m[self.prefix + "country_code"], e))
#gir = self._gi.region_by_addr(data)
#m["geo_region_code"] = gir['region']
#m["geo_region_name"] = gir['region_name']
yield m
def process(self, ctx, m):
data = ctx.interpolate(self.data, m)
m[self.prefix + "country_code"] = self._gi.country_code_by_addr(data)
m[self.prefix + "country_name"] = self._gi.country_name_by_addr(data) #.decode("latin1")
m[self.prefix + "cont_name"] = None
if m[self.prefix + "country_code"]:
try:
m[self.prefix + "cont_name"] = transformations.ccn_to_ctn(transformations.cca2_to_ccn(m[self.prefix + "country_code"])) #.decode("latin1")
except Exception as e:
if not self._extract_error and not ctx.debug2:
self._extract_error = True
logger.warning("Could not extract continent name from country code '%s' (reported only once per run) Error: %s" % (m[self.prefix + "country_code"], e))
#gir = self._gi.region_by_addr(data)
#m["geo_region_code"] = gir['region']
#m["geo_region_name"] = gir['region_name']
yield m
def check_prices():
output = ""
prices = parse_prices()
#provider_prices = load_provider(name)
providers = get_providers()
loaded_providers = {}
winners = get_winners()
better_deal = ""
countries = prices.keys()
countries.sort()
counters = defaultdict(int)
output += 'remember rs has no prices (new republic of serbia)\n'
for country in countries:
if country == 'rs':
# we have a warning above... known problem XXX
continue
for type in prices[country]:
current_rate, provider_code, ext = prices[country][type]
if provider_code in providers:
try:
country_name = transformations.ccn_to_cn(
transformations.cca2_to_ccn(country))
except:
country_name = 'Unknown'
if provider_code in loaded_providers:
provider_prices = loaded_providers[provider_code]
else:
names = providers[provider_code]
if names.__class__ == str:
names = [names]
all_prices = []
all_providers = set()
for name in names:
if name[:-1] in ['freevoipdeal']:
name = 'freevoipdeal'
if name[:-1] in ['budgetvoip']:
name = 'budgetvoip'
all_providers.add(name)
# XXX check if prices are the same for given country...
provider_prices = load_provider(name)
loaded_providers[provider_code] = provider_prices
try:
country_price = normalize_price(
provider_prices[country][type])
all_prices.append(country_price)
except:
traceback.print_exc()
output += "problem with %s %s %s %s\n" % (
name, country, type, name)
ok = True
if all_prices:
last_price = all_prices[0]
for aprice in all_prices:
if aprice != last_price:
output += country_name + " [" + country + "]\t" + type + " Price inconsistency %s %s\n" % (
str(all_prices), str(all_providers))
ok = False
else:
output += "No prices for %s %s %s %s\n" % (
name, country, type, str(all_providers))
if country in provider_prices and type in provider_prices[
country]:
current_rate = float(current_rate)
names = providers[provider_code]
provider_rate = normalize_price(
provider_prices[country][type])
if current_rate != provider_rate:
key = country + " " + type
if key in winners:
output += country_name + " [" + country + "]\t" + type + " %s -> %s %s %s\n" % (
str(current_rate), str(provider_rate),
str(providers[provider_code]), str(
winners[key]))
else:
output += country_name + " [" + country + "]\t" + type + " %s -> %s %s\n" % (
str(current_rate), str(provider_rate),
str(providers[provider_code]))
else:
key = country + " " + type
if key in winners:
winner_rate = normalize_price(winners[key][0])
if winner_rate < current_rate:
better_deal += "%s [%s]\t%s %s %s %s %s\n" % (
country_name, country, type, current_rate,
providers[provider_code], winner_rate,
str(winners[key][1]))
for winner in winners[key][1]:
counters[winner] += 1
else:
output += "%s %s not available\n" % (country, type)
else:
output += "unknown " + str(provider_code) + "\n"
output += "\n\nBETTER DEALS\n"
output += better_deal
output += "\nWINNER COUNTERS\n"
for winner in counters:
output += "<b>%s:</b> %s\n" % (winner, counters[winner])
return output
def country_expand(short_country):
code_num = transformations.cca2_to_ccn(short_country)
return transformations.ccn_to_cn(int(code_num))
def check_prices():
output = ""
prices = parse_prices()
#provider_prices = load_provider(name)
providers = get_providers()
loaded_providers = {}
winners = get_winners()
better_deal = ""
countries = prices.keys()
countries.sort()
counters = defaultdict(int)
for country in countries:
for type in prices[country]:
current_rate, provider_code, ext = prices[country][type]
if provider_code in providers:
try:
country_name = transformations.ccn_to_cn(transformations.cca2_to_ccn(country))
except:
country_name = 'Unknown'
if provider_code in loaded_providers:
provider_prices = loaded_providers[provider_code]
else:
names = providers[provider_code]
if names.__class__ == str:
names = [names]
all_prices = []
all_providers = set()
for name in names:
if name in ['freevoipdeal2', 'freevoipdeal3']:
name = 'freevoipdeal'
all_providers.add(name)
# XXX check if prices are the same for given country...
provider_prices = load_provider(name)
loaded_providers[provider_code] = provider_prices
try:
country_price = normalize_price(provider_prices[country][type])
all_prices.append(country_price)
except:
traceback.print_exc()
output += "problem with %s %s %s %s\n" % (name, country, type, name)
ok = True
if all_prices:
last_price = all_prices[0]
for aprice in all_prices:
if aprice != last_price:
output += country_name + " ["+country+"]\t"+type + " Price inconsistency %s %s\n" % (str(all_prices), str(all_providers))
ok = False
else:
output += "No prices for %s %s %s %s\n" % (name, country, type, str(all_providers))
if country in provider_prices and type in provider_prices[country]:
current_rate = float(current_rate)
names = providers[provider_code]
provider_rate = normalize_price(provider_prices[country][type])
if current_rate != provider_rate:
key = country + " " + type
if key in winners:
output += country_name + " ["+country+"]\t"+type + " %s -> %s %s %s\n" % (str(current_rate), str(provider_rate), str(providers[provider_code]), str(winners[key]))
else:
output += country_name + " ["+country+"]\t"+type + " %s -> %s %s\n" % (str(current_rate), str(provider_rate), str(providers[provider_code]))
else:
key = country + " " + type
if key in winners:
winner_rate = normalize_price(winners[key][0])
if winner_rate < current_rate:
better_deal += "%s [%s]\t%s %s %s %s %s\n" % (country_name, country, type, current_rate, providers[provider_code], winner_rate, str(winners[key][1]))
for winner in winners[key][1]:
counters[winner] += 1
else:
output += "%s %s not available\n" % (country, type)
else:
output += "unknown " + str(provider_code) + "\n"
output += "\n\nBETTER DEALS\n"
output += better_deal
output += "\nWINNER COUNTERS\n"
for winner in counters:
output += "<b>%s:</b> %s\n" % (winner, counters[winner])
return output
def get_location(query, format, api_key):
"""Get geographic data of a lab in a coherent way for all labs."""
# Play nice with the API...
sleep(1)
geolocator = OpenCage(api_key=api_key, timeout=10)
# Variables for storing the data
data = {
"city": None,
"address_1": None,
"postal_code": None,
"country": None,
"county": None,
"state": None,
"country_code": None,
"latitude": None,
"longitude": None,
"continent": None
}
road = ""
number = ""
# Default None values
location_data = {
"city": None,
"road": None,
"house_number": None,
"postcode": None,
"country": None,
"county": None,
"state": None,
"ISO_3166-1_alpha-2": None,
"country_code": None,
"lat": None,
"lng": None
}
# Reverse geocoding ... from coordinates to address
if format == "reverse":
# If the query (coordinates) is not empty
if query is None or len(query) < 3:
pass
else:
location = geolocator.reverse(query)
if location is not None:
location_data = location[0].raw[u'components']
location_data["lat"] = location[0].raw[u'geometry']["lat"]
location_data["lng"] = location[0].raw[u'geometry']["lng"]
# Direct geocoding ... from address to coordinates and full address
if format == "direct":
# If the query (address) is not empty
if query is None or len(query) < 3:
pass
else:
location = geolocator.geocode(query)
if location is not None:
location_data = location.raw[u'components']
location_data["lat"] = location.raw[u'geometry']["lat"]
location_data["lng"] = location.raw[u'geometry']["lng"]
# Extract the meaningful data
for component in location_data:
if component == "town" or component == "city":
data["city"] = location_data[component]
if component == "road":
road = location_data[component]
if component == "house_number":
number = location_data[component]
if component == "postcode":
data["postal_code"] = location_data[component]
if component == "country":
data["country"] = location_data[component]
if component == "county":
data["county"] = location_data[component]
if component == "state":
data["state"] = location_data[component]
if component == "ISO_3166-1_alpha-2":
data["country_code"] = location_data[component]
# The address need to be reconstructed
data["address_1"] = unicode(road) + " " + unicode(number)
data["latitude"] = location_data["lat"]
data["longitude"] = location_data["lng"]
# Format the country code to three letters
try:
country_data = transformations.cca2_to_ccn(data["country_code"])
data["country_code"] = transformations.ccn_to_cca3(country_data)
except:
data["country_code"] = None
# Get the continent
try:
country_data = transformations.cc_to_cn(data["country_code"])
data["continent"] = transformations.cn_to_ctn(country_data)
except:
data["continent"] = None
# Return the final data
return data
