def operator_code_to_full_name(code):
import mobile_codes
import re
h = HTMLParser.HTMLParser()
a = re.compile("^\d{5,}")
if not code or not a.match(code):
return code
mcc = code[:3]
mnc = code[3:]
try:
op = mobile_codes.mcc_mnc(mcc, mnc)
cn = mobile_codes.mcc(mcc)
try:
return u"{0} ({1}) - {2}".format(h.unescape(op.operator), mnc, cn[0].name)
except AttributeError:
return u"{0} ({1}) - {2}".format(h.unescape(op.operator), mnc, cn[0])
except KeyError:
try:
cn = mobile_codes.mcc(mcc)
except KeyError:
return u"Unknown ({0}/{1})".format(mcc, mnc)
try:
return u"{0} unknown ({1}/{2})".format(cn.name, mcc, mnc)
except AttributeError:
return u"Unknown ({0}/{1})".format(mcc, mnc)
def operator_code_to_full_name(code):
import mobile_codes
import re
h = HTMLParser.HTMLParser()
a = re.compile("^\d{5,}")
if not code or not a.match(code):
return code
mcc = code[:3]
mnc = code[3:]
try:
op = mobile_codes.mcc_mnc(mcc, mnc)
cn = mobile_codes.mcc(mcc)
try:
return u"{0} ({1}) - {2}".format(h.unescape(op.operator), mnc,
cn[0].name)
except AttributeError:
return u"{0} ({1}) - {2}".format(h.unescape(op.operator), mnc,
cn[0])
except KeyError:
try:
cn = mobile_codes.mcc(mcc)
except KeyError:
return u"Unknown ({0}/{1})".format(mcc, mnc)
try:
return u"{0} unknown ({1}/{2})".format(cn.name, mcc, mnc)
except AttributeError:
return u"Unknown ({0}/{1})".format(mcc, mnc)
def test_mcc_multiple_codes(self):
countries = mobile_codes.mcc(u'313')
self.assertEqual(len(countries), 1)
self.assertEqual(countries[0].mcc, [u'310', u'311', u'313', u'316'])
# We even get multiple countries with multiple MCC each
countries = mobile_codes.mcc(u'310')
self.assertTrue(len(countries) > 1)
for country in countries:
self.assertTrue(len(country.mcc) > 1)
def test_mcc_multiple_codes(self):
countries = mobile_codes.mcc(u'313')
self.assertEqual(len(countries), 1)
self.assertEqual(countries[0].mcc, [u'310', u'311', u'313', u'316'])
# We even get multiple countries with multiple MCC each
countries = mobile_codes.mcc(u'310')
self.assertTrue(len(countries) > 1)
for country in countries:
self.assertTrue(len(country.mcc) > 1)
def countries(session):
# we group by radio, mcc to take advantage of the index
rows = session.query(Cell.radio, Cell.mcc, func.count(Cell.id)).filter(
Cell.radio.in_([0, 1, 2, 3])).group_by(Cell.radio, Cell.mcc).all()
# reverse grouping by mcc, radio
codes = defaultdict(dict)
for row in rows:
codes[row[1]][row[0]] = row[2]
countries = {}
for code, item in codes.items():
names = [(c.name, c.alpha3) for c in mcc(str(code))]
multiple = bool(len(names) > 1)
for name, alpha3 in names:
country = {
'code': alpha3,
'name': name,
'multiple': multiple,
'total': 0,
'gsm': 0, 'cdma': 0, 'umts': 0, 'lte': 0,
}
for t, v in item.items():
country[RADIO_TYPE_INVERSE[t]] = int(v)
country['total'] = int(sum(item.values()))
if alpha3 not in countries:
countries[alpha3] = country
else:
# some countries like the US have multiple mcc codes,
# we merge them here
for k, v in country.items():
if isinstance(v, int):
countries[alpha3][k] += v
return sorted(countries.values(), key=itemgetter('name'))
def countries(session):
# we group by radio, mcc to take advantage of the index
rows = session.query(Cell.radio, Cell.mcc, func.count(Cell.id)).filter(
Cell.radio.in_([0, 1, 2, 3])).group_by(Cell.radio, Cell.mcc).all()
# reverse grouping by mcc, radio
codes = defaultdict(dict)
for row in rows:
codes[row[1]][row[0]] = row[2]
countries = {}
for code, item in codes.items():
try:
name = mcc(str(code)).name
except KeyError:
# we have some bogus networks in the database
continue
country = {
'name': name,
'total': 0,
'gsm': 0, 'cdma': 0, 'umts': 0, 'lte': 0,
}
for t, v in item.items():
country[RADIO_TYPE_INVERSE[t]] = int(v)
country['total'] = int(sum(item.values()))
if name not in countries:
countries[name] = country
else:
# some countries like the US have multiple mcc codes,
# we merge them here
for k, v in country.items():
if isinstance(v, int):
countries[name][k] += v
return sorted(countries.values(), key=itemgetter('name'))
def normalized_cell_measure_dict(d, measure_radio=-1):
"""
Returns a normalized copy of the provided cell-measure dict d,
or None if the dict was invalid.
"""
d = normalized_cell_dict(d, default_radio=measure_radio)
d = normalized_measure_dict(d)
location_is_in_country = geocalc.location_is_in_country
if d is not None:
# Lat/lon must be inside one of the bounding boxes for the MCC.
lat = float(d['lat'])
lon = float(d['lon'])
if not any([location_is_in_country(lat, lon, c.alpha2, 1)
for c in mobile_codes.mcc(str(d['mcc']))]):
d = None
if d is None:
return None
if 'asu' in d and 'signal' in d:
# some clients send us a dBm value in the asu field, move it
# over to the signal field before hitting validation
if d['asu'] < -1 and d['signal'] == 0:
d['signal'] = d['asu']
d['asu'] = -1
return normalized_dict(
d, dict(asu=(0, 31, -1),
signal=(-200, -1, 0),
ta=(0, 63, 0)))
def normalized_cell_measure_dict(d, measure_radio=-1):
"""
Returns a normalized copy of the provided cell-measure dict d,
or None if the dict was invalid.
"""
d = normalized_cell_dict(d, default_radio=measure_radio)
d = normalized_measure_dict(d)
location_is_in_country = geocalc.location_is_in_country
if d is not None:
# Lat/lon must be inside one of the bounding boxes for the MCC.
lat = float(d['lat'])
lon = float(d['lon'])
if not any([
location_is_in_country(lat, lon, c.alpha2, 1)
for c in mobile_codes.mcc(str(d['mcc']))
]):
d = None
if d is None:
return None
if 'asu' in d and 'signal' in d:
# some clients send us a dBm value in the asu field, move it
# over to the signal field before hitting validation
if d['asu'] < -1 and d['signal'] == 0:
d['signal'] = d['asu']
d['asu'] = -1
return normalized_dict(
d, dict(asu=(0, 97, -1), signal=(-150, -1, 0), ta=(0, 63, 0)))
def regions_for_mcc(self, mcc, metadata=False):
"""
Return a list of region codes matching the passed in
mobile country code.
If the metadata argument is set to True, returns a list of
region instances containing additional metadata instead.
The return list is filtered by the set of recognized
region codes present in the GENC dataset.
"""
codes = [region.alpha2 for region in mobile_codes.mcc(str(mcc))]
# map mcc region codes to genc region codes
codes = [MCC_TO_GENC_MAP.get(code, code) for code in codes]
valid_codes = set(codes).intersection(self._valid_regions)
if not metadata:
return list(valid_codes)
result = []
for code in valid_codes:
region = genc.region_by_alpha2(code)
if region is not None:
result.append(Region(
code=region.alpha2,
name=region.name,
radius=self.region_max_radius(code)))
return result
def regions_for_mcc(self, mcc, metadata=False):
"""
Return a list of region codes matching the passed in
mobile country code.
If the metadata argument is set to True, returns a list of
dictionaries containing additional metadata instead.
The return list is filtered by the set of recognized
region codes present in the GENC dataset.
"""
codes = [region.alpha2 for region in mobile_codes.mcc(str(mcc))]
# map mcc region codes to genc region codes
codes = [MCC_GENC_SHAPEFILE_MAP.get(code, code) for code in codes]
valid_codes = set(codes).intersection(self._valid_regions)
if not metadata:
return list(valid_codes)
result = []
for code in valid_codes:
region = genc.region_by_alpha2(code)
if region is not None:
result.append(
Region(code=region.alpha2,
name=region.name,
radius=self.region_max_radius(code)))
return result
def get_carrier(self, networkcode):
#Use case 1:- if the network code exists then return the carrier
#Use case 2: if the network code doesn't exist extract the country and use it to get the default carrier
#Use case 3:- Return any carrier if no carrier is specified as default
carrier = None
try:
carrier = USSDCarrier.objects.get(code=networkcode)
except USSDCarrier.DoesNotExist:
import mobile_codes
mcc = networkcode[0:3]
try:
country_iso = mobile_codes.mcc(mcc)[0].alpha2 #tested for Uganda,Zambia,Nigeria
except Exception as e:
logging.exception(e.message)
else:
try:
carrier = USSDCarrier.objects.get(country__iso_2_abbreviation=country_iso,country_default=True)
except USSDCarrier.DoesNotExist:
try:
carrier = USSDCarrier.objects.filter(country__iso_2_abbreviation=country_iso)[0]
except IndexError:
pass
return carrier
def test_locate_finds_country_from_mcc(self):
country = mobile_codes.mcc("235")[0]
cell = CellFactory.build(mcc=235)
query = self.model_query(cells=[cell])
location = self.provider.locate(query)
self.check_model_location(location, country)
def geoip_and_best_guess_country_codes(cell_keys, api_name, client_addr,
geoip_db):
"""
Return (geoip, alpha2) where geoip is the result of a GeoIP lookup
and alpha2 is a best-guess ISO 3166 alpha2 country code. The country
code guess uses both GeoIP and cell MCCs, preferring GeoIP. Return
None for either field if no data is available.
"""
stats_client = get_stats_client()
geoip = None
if client_addr and geoip_db is not None:
geoip = geoip_db.geoip_lookup(client_addr)
cell_countries = []
cell_mccs = set()
for cell_key in cell_keys:
for c in mobile_codes.mcc(str(cell_key.mcc)):
cell_countries.append(c.alpha2)
cell_mccs.add(cell_key.mcc)
if len(cell_mccs) > 1:
stats_client.incr('%s.anomaly.multiple_mccs' % api_name)
if geoip:
# GeoIP always wins if we have it.
accuracy, city = radius_from_geoip(geoip)
if city:
stats_client.incr('%s.geoip_city_found' % api_name)
else:
stats_client.incr('%s.geoip_country_found' % api_name)
if geoip['country_code'] not in cell_countries:
if cell_countries:
stats_client.incr('%s.anomaly.geoip_mcc_mismatch' % api_name)
# Only use the GeoIP country as an additional possible match,
# but retain the cell countries as a likely match as well.
cell_countries.append(geoip['country_code'])
stats_client.incr('%s.country_from_geoip' % api_name)
geoip_res = {
'lat': geoip['latitude'],
'lon': geoip['longitude'],
'accuracy': accuracy
}
return (geoip_res, most_common_elements(cell_countries))
else:
stats_client.incr('%s.no_geoip_found' % api_name)
# Pick the most-commonly-occurring country codes if we got any
cc = most_common_elements(cell_countries)
if cc:
stats_client.incr('%s.country_from_mcc' % api_name)
return (None, cc)
stats_client.incr('%s.no_country' % api_name)
return (None, [])
def geoip_and_best_guess_country_codes(cell_keys, api_name,
client_addr, geoip_db):
"""
Return (geoip, alpha2) where geoip is the result of a GeoIP lookup
and alpha2 is a best-guess ISO 3166 alpha2 country code. The country
code guess uses both GeoIP and cell MCCs, preferring GeoIP. Return
None for either field if no data is available.
"""
stats_client = get_stats_client()
geoip = None
if client_addr and geoip_db is not None:
geoip = geoip_db.geoip_lookup(client_addr)
cell_countries = []
cell_mccs = set()
for cell_key in cell_keys:
for c in mobile_codes.mcc(str(cell_key.mcc)):
cell_countries.append(c.alpha2)
cell_mccs.add(cell_key.mcc)
if len(cell_mccs) > 1:
stats_client.incr('%s.anomaly.multiple_mccs' % api_name)
if geoip:
# GeoIP always wins if we have it.
accuracy, city = radius_from_geoip(geoip)
if city:
stats_client.incr('%s.geoip_city_found' % api_name)
else:
stats_client.incr('%s.geoip_country_found' % api_name)
if geoip['country_code'] not in cell_countries:
if cell_countries:
stats_client.incr('%s.anomaly.geoip_mcc_mismatch' % api_name)
# Only use the GeoIP country as an additional possible match,
# but retain the cell countries as a likely match as well.
cell_countries.append(geoip['country_code'])
stats_client.incr('%s.country_from_geoip' % api_name)
geoip_res = {
'lat': geoip['latitude'],
'lon': geoip['longitude'],
'accuracy': accuracy
}
return (geoip_res, most_common_elements(cell_countries))
else:
stats_client.incr('%s.no_geoip_found' % api_name)
# Pick the most-commonly-occurring country codes if we got any
cc = most_common_elements(cell_countries)
if cc:
stats_client.incr('%s.country_from_mcc' % api_name)
return (None, cc)
stats_client.incr('%s.no_country' % api_name)
return (None, [])
def test_mcc_multiple_countries_one_code(self):
countries = mobile_codes.mcc("313") # MCC used in US and Puerto Rico
assert len(countries) == 2
actual = {country.alpha2: country.mcc for country in countries}
assert actual == {
"PR": ["310", "313", "330"],
"US": ["310", "311", "312", "313", "314", "315", "316"],
}
def _query_database(self, cell_keys):
countries = []
for key in cell_keys:
countries.extend(mobile_codes.mcc(str(key.mcc)))
if len(set([c.alpha2 for c in countries])) != 1:
# refuse to guess country if there are multiple choices
return []
return countries[0]
def test_locate_finds_country_from_mcc(self):
country = mobile_codes.mcc(str(CANADA_MCC))[0]
cell_key = {'mcc': CANADA_MCC, 'mnc': 1, 'lac': 1}
location = self.provider.locate(
{'cell': [dict(cid=1, radio=Radio.gsm.name, **cell_key)]})
self.assertEqual(type(location), Country)
self.assertEqual(location.country_code, country.alpha2)
self.assertEqual(location.country_name, country.name)
def _query_database(self, cell_keys):
countries = []
for key in cell_keys:
countries.extend(mobile_codes.mcc(str(key.mcc)))
if len(set([c.alpha2 for c in countries])) != 1:
# refuse to guess country if there are multiple choices
return []
return countries[0]
def create_transaction(self, generic_seller, generic_product,
provider_product, product_name, transaction_uuid,
prices, user_uuid, application_size, source,
icon_url, mcc=None, mnc=None):
try:
# Do a sanity check to make sure we're actually on a Boku network.
self.network_data[(mcc, mnc)]
except KeyError:
raise self.TransactionError('Unknown Boku network: '
'mcc={mcc}; mnc={mnc}'
.format(mcc=mcc, mnc=mnc))
country = mobile_codes.mcc(mcc)
# TODO: consider using get_price_country here?
mcc_region = COUNTRIES[mcc]
price = None
for mktpl_price in prices:
# Given a list of all prices + currencies for this price point,
# send Boku the one that matches the user's network/region.
if mktpl_price['region'] == mcc_region:
price = mktpl_price['price']
break
if not price:
log.error('No Boku price for region {r}: mcc={mcc}; mnc={mnc} '
'in prices {pr}'.format(mcc=mcc, mnc=mnc,
r=mcc_region, pr=prices))
raise self.TransactionError(
'Could not find a price for region {r}: mcc={mcc}; mnc={mnc}'
.format(mcc=mcc, mnc=mnc, r=mcc_region))
provider_trans = self.api.transactions.post({
'forward_url': absolutify(reverse('provider.wait_to_finish',
args=[self.name])),
'callback_url': absolutify(reverse('provider.notification',
args=[self.name])),
'country': country.alpha2,
# TODO: figure out error callbacks in bug 987843.
#'error_url': absolutify(reverse('provider.error',
# args=[self.name])),
'price': price,
'seller_uuid': generic_seller['uuid'],
'transaction_uuid': transaction_uuid,
'user_uuid': user_uuid,
})
log.info('{pr}: made provider trans {trans}'
.format(pr=self.name, trans=provider_trans))
trans = self.slumber.generic.transaction.post({
'provider': solitude_const.PROVIDERS[self.name],
'seller_product': generic_product['resource_uri'],
'source': solitude_const.PROVIDERS[self.name],
'status': solitude_const.STATUS_PENDING,
'type': solitude_const.TYPE_PAYMENT,
'uuid': transaction_uuid,
})
log.info('{pr}: made solitude trans {trans}'
.format(pr=self.name, trans=trans))
return provider_trans['transaction_id'], provider_trans['buy_url']
def validator(self, schema, data):
super(ValidCellObservationSchema, self).validator(schema, data)
in_country = False
for code in mobile_codes.mcc(str(data['mcc'])):
in_country = in_country or geocalc.location_is_in_country(
data['lat'], data['lon'], code.alpha2, 1)
if not in_country:
raise Invalid(schema, ('Lat/lon must be inside one of '
'the bounding boxes for the MCC'))
def test_ambiguous_mcc(self):
countries = mobile_codes.mcc('234')
cell = CellFactory.build(mcc=234)
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_result(results, countries)
self.check_stats(counter=[
(self.api_type + '.source',
['key:test', 'country:none', 'source:internal',
'accuracy:low', 'status:hit']),
])
def validator(self, schema, data):
super(ValidCellObservationSchema, self).validator(schema, data)
in_country = False
for code in mobile_codes.mcc(str(data['mcc'])):
in_country = in_country or geocalc.location_is_in_country(
data['lat'], data['lon'], code.alpha2, 1)
if not in_country:
raise colander.Invalid(schema, (
'Lat/lon must be inside one of '
'the bounding boxes for the MCC'))
def validator(self, node, cstruct):
super(ValidCellObservationSchema, self).validator(node, cstruct)
in_country = False
for code in mobile_codes.mcc(str(cstruct['mcc'])):
in_country = in_country or geocalc.country_matches_location(
cstruct['lat'], cstruct['lon'], code.alpha2, 1)
if not in_country:
raise colander.Invalid(node, (
'Lat/lon must be inside one of '
'the bounding boxes for the MCC'))
def create_transaction(self, generic_seller, generic_product,
provider_product, product_name, transaction_uuid,
prices, user_uuid, application_size, source,
icon_url, mcc=None, mnc=None):
try:
data = self.network_data[(mcc, mnc)]
except KeyError:
raise self.TransactionError('Unknown network: mcc={mcc}; mnc={mnc}'
.format(mcc=mcc, mnc=mnc))
country = mobile_codes.mcc(mcc)
price = None
for pr in prices:
# Given a list of all prices + currencies for this price point,
# send Boku the one that matches the user's network.
if pr['currency'] == data['currency']:
price = pr['price']
break
if not price:
raise self.TransactionError(
'Could not find a price for currency {cur}; '
'mcc={mcc}; mnc={mnc}'.format(cur=data['currency'],
mcc=mcc, mnc=mnc))
provider_trans = self.api.transactions.post({
'callback_url': absolutify(reverse('provider.success',
args=[self.name])),
'country': country.alpha2,
# TODO: figure out error callbacks in bug 987843.
#'error_url': absolutify(reverse('provider.error',
# args=[self.name])),
'price': price,
'seller_uuid': generic_seller['uuid'],
'transaction_uuid': transaction_uuid,
'user_uuid': user_uuid,
})
log.info('{pr}: made provider trans {trans}'
.format(pr=self.name, trans=provider_trans))
trans = self.slumber.generic.transaction.post({
'provider': solitude_const.PROVIDERS[self.name],
'seller_product': generic_product['resource_uri'],
'source': solitude_const.PROVIDERS[self.name],
'status': solitude_const.STATUS_PENDING,
'type': solitude_const.TYPE_PAYMENT,
'uuid': transaction_uuid,
})
log.info('{pr}: made solitude trans {trans}'
.format(pr=self.name, trans=trans))
return provider_trans['transaction_id'], provider_trans['buy_url']
def main():
settings = read_config().get_map('ichnaea')
db = Database(settings['db_slave'])
session = db.session()
bad = []
offset = 0
count = 10000
results = True
while results:
results = False
r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, "
"total_measures from cell where "
"lat is not null and lon is not null and "
"mcc not in (1, 260) "
"order by id limit %d offset %d" %
(count, offset))
offset += count
for row in r:
results = True
(id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row
ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))]
if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]):
if ccs:
s = ",".join(ccs)
else:
continue
bad.append(dict(
type='Feature',
properties=dict(
mcc=mcc,
mnc=mnc,
lac=lac,
cid=cid,
radio=radio,
total_measures=total_measures,
countries=s),
geometry=dict(
type='Point',
coordinates=[lon, lat])))
json.dump(dict(type='FeatureCollection',
features=bad),
sys.stdout,
indent=True)
def regions(session):
# We group by radio, mcc to take advantage of the index
# and explicitly specify a small list of all valid radio values
# to get mysql to actually use the index.
radios = set([radio for radio in Radio])
rows = session.query(Cell.radio, Cell.mcc, func.count()).filter(
Cell.radio.in_(radios)).group_by(Cell.radio, Cell.mcc).all()
# reverse grouping by mcc, radio
mccs = defaultdict(dict)
for row in rows:
mccs[row.mcc][row.radio] = row[2]
regions = {}
for mcc, item in mccs.items():
iso_codes = [rec.alpha2 for rec in mobile_codes.mcc(str(mcc))]
multiple = bool(len(iso_codes) > 1)
for alpha2 in iso_codes:
name = iso3166.countries_by_alpha2[alpha2].apolitical_name
region = {
'code': alpha2,
'name': name,
'order': transliterate(name[:10].lower()),
'multiple': multiple,
'total': 0,
'gsm': 0, 'cdma': 0, 'umts': 0, 'lte': 0,
}
for radio, value in item.items():
radio_name = radio.name
if radio_name == 'wcdma':
radio_name = 'umts'
region[radio_name] = int(value)
region['total'] = int(sum(item.values()))
if alpha2 not in regions:
regions[alpha2] = region
else:
# some regions like the US have multiple mcc codes,
# we merge them here
for radio_name, value in region.items():
if isinstance(value, int):
regions[alpha2][radio_name] += value
return sorted(regions.values(), key=itemgetter('name'))
def main():
settings = read_config().get_map('ichnaea')
db = Database(settings['db_slave'])
session = db.session()
bad = []
offset = 0
count = 10000
results = True
while results:
results = False
r = session.execute("select id, lat, lon, mcc, mnc, lac, cid, radio, "
"total_measures from cell where "
"lat is not null and lon is not null and "
"mcc not in (1, 260) "
"order by id limit %d offset %d" % (count, offset))
offset += count
for row in r:
results = True
(id, lat, lon, mcc, mnc, lac, cid, radio, total_measures) = row
ccs = [c.alpha2 for c in mobile_codes.mcc(str(mcc))]
if not any([location_is_in_country(lat, lon, c, 1) for c in ccs]):
if ccs:
s = ",".join(ccs)
else:
continue
bad.append(
dict(type='Feature',
properties=dict(mcc=mcc,
mnc=mnc,
lac=lac,
cid=cid,
radio=radio,
total_measures=total_measures,
countries=s),
geometry=dict(type='Point', coordinates=[lon, lat])))
json.dump(dict(type='FeatureCollection', features=bad),
sys.stdout,
indent=True)
def countries(session):
# We group by radio, mcc to take advantage of the index
# and explicitly specify a small list of all valid radio values
# to get mysql to actually use the index.
radios = [v for v in RADIO_TYPE.values() if v >= 0]
rows = session.query(Cell.radio, Cell.mcc, func.count()).filter(
Cell.radio.in_(radios)).group_by(Cell.radio, Cell.mcc).all()
# reverse grouping by mcc, radio
codes = defaultdict(dict)
for row in rows:
codes[row[1]][row[0]] = row[2]
countries = {}
for code, item in codes.items():
names = [(c.name, c.alpha3) for c in mcc(str(code))]
multiple = bool(len(names) > 1)
for name, alpha3 in names:
country = {
'code': alpha3,
'name': name,
'order': transliterate(name[:10].lower()),
'multiple': multiple,
'total': 0,
'gsm': 0,
'cdma': 0,
'umts': 0,
'lte': 0,
}
for t, v in item.items():
country[RADIO_TYPE_INVERSE[t]] = int(v)
country['total'] = int(sum(item.values()))
if alpha3 not in countries:
countries[alpha3] = country
else:
# some countries like the US have multiple mcc codes,
# we merge them here
for k, v in country.items():
if isinstance(v, int):
countries[alpha3][k] += v
return sorted(countries.values(), key=itemgetter('name'))
def countries(session):
# We group by radio, mcc to take advantage of the index
# and explicitly specify a small list of all valid radio values
# to get mysql to actually use the index.
radios = set([radio for radio in Radio])
rows = session.query(Cell.radio, Cell.mcc, func.count()).filter(
Cell.radio.in_(radios)).group_by(Cell.radio, Cell.mcc).all()
# reverse grouping by mcc, radio
codes = defaultdict(dict)
for row in rows:
codes[row[1]][row[0]] = row[2]
countries = {}
for code, item in codes.items():
names = [(c.name, c.alpha3) for c in mcc(str(code))]
multiple = bool(len(names) > 1)
for name, alpha3 in names:
country = {
'code': alpha3,
'name': name,
'order': transliterate(name[:10].lower()),
'multiple': multiple,
'total': 0,
'gsm': 0, 'cdma': 0, 'umts': 0, 'lte': 0,
}
for radio, value in item.items():
country[radio.name] = int(value)
country['total'] = int(sum(item.values()))
if alpha3 not in countries:
countries[alpha3] = country
else:
# some countries like the US have multiple mcc codes,
# we merge them here
for radio_name, value in country.items():
if isinstance(value, int):
countries[alpha3][radio_name] += value
return sorted(countries.values(), key=itemgetter('name'))
def countries(session):
# we group by radio, mcc to take advantage of the index
rows = session.query(Cell.radio, Cell.mcc, func.count(Cell.id)).filter(
Cell.radio.in_([0, 1, 2, 3])).group_by(Cell.radio, Cell.mcc).all()
# reverse grouping by mcc, radio
codes = defaultdict(dict)
for row in rows:
codes[row[1]][row[0]] = row[2]
countries = {}
for code, item in codes.items():
try:
name = mcc(str(code)).name
except KeyError:
# we have some bogus networks in the database
continue
country = {
'name': name,
'total': 0,
'gsm': 0,
'cdma': 0,
'umts': 0,
'lte': 0,
}
for t, v in item.items():
country[RADIO_TYPE_INVERSE[t]] = int(v)
country['total'] = int(sum(item.values()))
if name not in countries:
countries[name] = country
else:
# some countries like the US have multiple mcc codes,
# we merge them here
for k, v in country.items():
if isinstance(v, int):
countries[name][k] += v
return sorted(countries.values(), key=itemgetter('name'))
def search(self, query):
results = ResultList()
codes = set()
for cell in list(query.cell) + list(query.cell_area):
codes.add(cell.mcc)
countries = []
for code in codes:
countries.extend(mobile_codes.mcc(str(code)))
for country in countries:
country_code = country.alpha2
results.add(self.result_type(
country_code=country_code,
country_name=country.name,
accuracy=geoip_accuracy(country_code)))
if len(results):
query.emit_source_stats(self.source, results[0])
else:
results.add(self.result_type())
return results
def test_mcc_multiple_countries(self):
countries = mobile_codes.mcc(u'505')
self.assertEqual(len(countries), 2)
def test_mcc_fail(self):
countries = mobile_codes.mcc("000")
assert len(countries) == 0
def test_mcc_multiple_countries(self):
countries = mobile_codes.mcc("505")
assert len(countries) == 4
actual = {country.alpha2: country.mcc for country in countries}
assert actual == {"AU": "505", "CX": "505", "CC": "505", "NF": "505"}
def test_mcc_fail(self):
countries = mobile_codes.mcc(u'000')
self.assertEqual(len(countries), 0)
def test_mcc_multiple_codes_one_country(self):
countries = mobile_codes.mcc("312") # US-only MCC
assert len(countries) == 1
assert countries[0].mcc == [
"310", "311", "312", "313", "314", "315", "316"
]
def test_mcc_fail(self):
countries = mobile_codes.mcc(u'000')
self.assertEqual(len(countries), 0)
def test_mcc(self):
countries = mobile_codes.mcc(u'302')
self.assertEqual(len(countries), 1)
self.assertEqual(countries[0].mcc, u'302')
def test_mcc_multiple_countries(self):
countries = mobile_codes.mcc(u'505')
self.assertEqual(len(countries), 2)
def test_mcc(self):
countries = mobile_codes.mcc(u'302')
self.assertEqual(len(countries), 1)
self.assertEqual(countries[0].mcc, u'302')
data[k] = v.replace(data['created'], date)
for k, v in data.items():
date = datetime.datetime.utcfromtimestamp(
modified).strftime('%Y-%m-%d %H:%M:%S')
data[k] = v.replace(data['updated'], date)
'''for k,v in data.items():
data[k] = v.replace(data['radio'], '0')'''
#print data
last_seen = datetime.datetime.utcfromtimestamp(
modified).strftime('%Y-%m-%d')
#print last_seen
code_num = str(data['mcc'])
code_alpha = mobile_codes.mcc(code_num)
try:
region = code_alpha[0].alpha2
except IndexError:
region = None
print("mcc = %s skipped!" % (code_num))
#print region
lat = float(data['lat'])
lon = float(data['lon'])
radius = float(data['range'])
if radius > 0:
max_lat, min_lat, max_lon, min_lon = geocalc.bbox(
lat, lon, radius)
def create_transaction(self,
generic_seller,
generic_product,
provider_product,
product_name,
transaction_uuid,
prices,
user_uuid,
application_size,
source,
icon_url,
mcc=None,
mnc=None):
try:
# Do a sanity check to make sure we're actually on a Boku network.
self.network_data[(mcc, mnc)]
except KeyError:
raise self.TransactionError('Unknown Boku network: '
'mcc={mcc}; mnc={mnc}'.format(mcc=mcc,
mnc=mnc))
country = mobile_codes.mcc(mcc)
# TODO: consider using get_price_country here?
mcc_region = COUNTRIES[mcc]
price = None
for mktpl_price in prices:
# Given a list of all prices + currencies for this price point,
# send Boku the one that matches the user's network/region.
if mktpl_price['region'] == mcc_region:
price = mktpl_price['price']
break
if not price:
log.error('No Boku price for region {r}: mcc={mcc}; mnc={mnc} '
'in prices {pr}'.format(mcc=mcc,
mnc=mnc,
r=mcc_region,
pr=prices))
raise self.TransactionError(
'Could not find a price for region {r}: mcc={mcc}; mnc={mnc}'.
format(mcc=mcc, mnc=mnc, r=mcc_region))
provider_trans = self.api.transactions.post({
'forward_url':
absolutify(reverse('provider.wait_to_finish', args=[self.name])),
'callback_url':
absolutify(reverse('provider.notification', args=[self.name])),
'country':
country.alpha2,
# TODO: figure out error callbacks in bug 987843.
#'error_url': absolutify(reverse('provider.error',
# args=[self.name])),
'price':
price,
'seller_uuid':
generic_seller['uuid'],
'transaction_uuid':
transaction_uuid,
'user_uuid':
user_uuid,
})
log.info('{pr}: made provider trans {trans}'.format(
pr=self.name, trans=provider_trans))
trans = self.slumber.generic.transaction.post({
'provider':
solitude_const.PROVIDERS[self.name],
'seller_product':
generic_product['resource_uri'],
'source':
solitude_const.PROVIDERS[self.name],
'status':
solitude_const.STATUS_PENDING,
'type':
solitude_const.TYPE_PAYMENT,
'uuid':
transaction_uuid,
})
log.info('{pr}: made solitude trans {trans}'.format(pr=self.name,
trans=trans))
return provider_trans['transaction_id'], provider_trans['buy_url']
def create_transaction(self,
generic_seller,
generic_product,
provider_product,
product_name,
transaction_uuid,
prices,
user_uuid,
application_size,
source,
icon_url,
mcc=None,
mnc=None):
try:
data = self.network_data[(mcc, mnc)]
except KeyError:
raise self.TransactionError(
'Unknown network: mcc={mcc}; mnc={mnc}'.format(mcc=mcc,
mnc=mnc))
country = mobile_codes.mcc(mcc)
price = None
for pr in prices:
# Given a list of all prices + currencies for this price point,
# send Boku the one that matches the user's network.
if pr['currency'] == data['currency']:
price = pr['price']
break
if not price:
raise self.TransactionError(
'Could not find a price for currency {cur}; '
'mcc={mcc}; mnc={mnc}'.format(cur=data['currency'],
mcc=mcc,
mnc=mnc))
provider_trans = self.api.transactions.post({
'callback_url':
absolutify(reverse('provider.success', args=[self.name])),
'country':
country.alpha2,
# TODO: figure out error callbacks in bug 987843.
#'error_url': absolutify(reverse('provider.error',
# args=[self.name])),
'price':
price,
'seller_uuid':
generic_seller['uuid'],
'transaction_uuid':
transaction_uuid,
'user_uuid':
user_uuid,
})
log.info('{pr}: made provider trans {trans}'.format(
pr=self.name, trans=provider_trans))
trans = self.slumber.generic.transaction.post({
'provider':
solitude_const.PROVIDERS[self.name],
'seller_product':
generic_product['resource_uri'],
'source':
solitude_const.PROVIDERS[self.name],
'status':
solitude_const.STATUS_PENDING,
'type':
solitude_const.TYPE_PAYMENT,
'uuid':
transaction_uuid,
})
log.info('{pr}: made solitude trans {trans}'.format(pr=self.name,
trans=trans))
return provider_trans['transaction_id'], provider_trans['buy_url']
def metadata(meta_info, expconfig):
"""Seperate process that attach to the ZeroMQ socket as a subscriber.
Will listen forever to messages with topic defined in topic and update
the meta_info dictionary (a Manager dict).
"""
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect(expconfig['zmqport'])
socket.setsockopt(zmq.SUBSCRIBE, bytes(expconfig['modem_metadata_topic']))
socket.setsockopt(zmq.SUBSCRIBE, bytes(expconfig['gps_metadata_topic']))
while True:
data = socket.recv()
try:
topic = data.split(" ", 1)[0]
msg = json.loads(data.split(" ", 1)[1])
if topic.startswith(expconfig['modem_metadata_topic']):
if expconfig['verbosity'] > 2:
print("Got a modem message for {}, using interface {}"
).format(msg['Operator'], msg['InterfaceName'])
# use iccid as unique identifier for the interface as interface names may change
iccid = msg['ICCID']
# store extra info in metadata
try:
meta_info['modem']['extra_ipaddress'] = msg['IPAddress']
meta_info['modem']['extra_imsi'] = msg['IMSI']
meta_info['modem']['extra_location'] = mobile_codes.mcc(
meta_info['modem']['extra_imsi'][0:3])[0][0]
except Exception:
print(
"Could update extra information, appending instead: {}"
.format(e))
# get internal interface from previous metadata update
try:
prev_internal_interface = meta_info['modems'][iccid][
'internalinterface']
except KeyError:
prev_internal_interface = msg['internalinterface']
internal_interface = msg['internalinterface']
if check_modem_meta(
msg) and internal_interface == prev_internal_interface:
meta_info['interface_events'][internal_interface].set()
else:
print 'An interface has been remapped, killing %s and %s' % (
internal_interface, prev_internal_interface)
# clear previous internal interface
try:
meta_info['interface_events'][
prev_internal_interface].clear()
except KeyError:
pass
# clear new interface since a remapping has happened
try:
meta_info['interface_events'][
internal_interface].clear()
except KeyError:
pass
# store metadata in dictionary
try:
meta_info['modem'][iccid].update(msg)
except KeyError:
meta_info['modem'][iccid] = msg
if topic.startswith(expconfig['gps_metadata_topic']):
if expconfig['verbosity'] > 2:
print("Got a gps message with seq nr {}").format(
msg["SequenceNumber"])
meta_info['gps'].append(msg)
if expconfig['verbosity'] > 2:
print "zmq message", topic, msg
except Exception as e:
if expconfig['verbosity'] > 0:
print(
"Cannot get metadata in template container {}, {}").format(
e, expconfig['guid'])
pass
def test_mcc(self):
countries = mobile_codes.mcc("302")
assert len(countries) == 1
assert countries[0].mcc == "302"
