def generate_gprs_events(start_timestamp, end_timestamp):
"""Create GPRS events from data in the GPRS DB.
Records that were generated between the specified timestamp will become
events. One event is created per IMSI (not one event per record).
Args:
start_timestamp: seconds since epoch
end_timestamp: seconds since epoch
"""
gprs_db = gprs_database.GPRSDB()
# First organize the records by IMSI.
sorted_records = {}
for record in gprs_db.get_records(start_timestamp, end_timestamp):
if record['imsi'] not in sorted_records:
sorted_records[record['imsi']] = []
sorted_records[record['imsi']].append(record)
# Now analyze all records that we have for each IMSI.
for imsi in sorted_records:
up_bytes = sum(
[r['uploaded_bytes_delta'] for r in sorted_records[imsi]])
down_bytes = sum(
[r['downloaded_bytes_delta'] for r in sorted_records[imsi]])
# Do not make an event if the byte deltas are unchanged.
if up_bytes == 0 and down_bytes == 0:
continue
# For now, GPRS is free for subscribers.
cost = 0
reason = 'gprs_usage: %s uploaded, %s downloaded' % (
humanize.naturalsize(up_bytes), humanize.naturalsize(down_bytes))
timespan = int(end_timestamp - start_timestamp)
events.create_gprs_event(imsi, cost, reason, up_bytes, down_bytes,
timespan)
def setUp(self):
# Connect to the GPRSDB.
self.gprs_db = gprs_database.GPRSDB()
# Add some records to the GPRSDB.
self.now = time.time()
records = [
(psycopg2.TimestampFromTicks(self.now - 60), 'IMSI901550000000084',
'192.168.99.1', 100, 200, 100, 200),
(psycopg2.TimestampFromTicks(self.now - 30), 'IMSI901550000000084',
'192.168.99.1', 300, 500, 200, 300),
(psycopg2.TimestampFromTicks(self.now - 10), 'IMSI901550000000084',
'192.168.99.1', 700, 600, 400, 100),
(psycopg2.TimestampFromTicks(self.now - 5), 'IMSI901550000000084',
'192.168.99.1', 750, 625, 50, 25),
]
schema = ('record_timestamp, imsi, ipaddr, uploaded_bytes,'
' downloaded_bytes, uploaded_bytes_delta,'
' downloaded_bytes_delta')
connection = psycopg2.connect(host='localhost',
database='endaga',
user=PG_USER,
password=PG_PASSWORD)
with connection.cursor() as cursor:
for record in records:
values = "%s, '%s', '%s', %s, %s, %s, %s" % record
command = 'insert into gprs_records (%s) values(%s)' % (schema,
values)
cursor.execute(command)
connection.commit()
def setUpClass(cls):
# Monkeypatch Subscriber.
cls.original_subscriber = utilities.subscriber
cls.mock_subscriber = mocks.MockSubscriber()
utilities.subscriber = cls.mock_subscriber
# Connect to the GPRSDB.
cls.gprs_db = gprs_database.GPRSDB()
def clean_old_gprs_records(timestamp):
"""Remove records from the GPRS DB that are older than timestamp.
We just do this to prevent the table from growing without bound. And we
don't delete GPRS records immediately after their conversion to events in
case we want them for analysis later.
Args:
timestamp: seconds since epoch
"""
gprs_db = gprs_database.GPRSDB()
gprs_db.delete_records(timestamp)
def setUpClass(cls):
# Monkeypatch Subscriber so sub balance lookups succeed.
cls.original_subscriber = utilities.subscriber
cls.mock_subscriber = mocks.MockSubscriber()
utilities.subscriber = cls.mock_subscriber
subscriber.create_subscriber('IMSI901550000000084', '5551234')
subscriber.create_subscriber('IMSI901550000000082', '5551235')
# Connect to the GPRSDB and EventStore.
cls.gprs_db = gprs_database.GPRSDB()
cls.event_store = events.EventStore()
# Add some records to the GPRSDB. The method we're testing should
# extract these records and create events in the EventStore.
cls.now = time.time()
records = [
(psycopg2.TimestampFromTicks(cls.now - 120), 'IMSI901550000000084',
'192.168.99.1', 50, 80, 50, 80),
(psycopg2.TimestampFromTicks(cls.now - 60), 'IMSI901550000000084',
'192.168.99.1', 50, 80, 0, 0),
(psycopg2.TimestampFromTicks(cls.now - 30), 'IMSI901550000000084',
'192.168.99.1', 300, 500, 250, 420),
(psycopg2.TimestampFromTicks(cls.now - 10), 'IMSI901550000000084',
'192.168.99.1', 700, 600, 400, 100),
(psycopg2.TimestampFromTicks(cls.now - 5), 'IMSI901550000000084',
'192.168.99.1', 750, 625, 50, 25),
# Create events for a different IMSI.
(psycopg2.TimestampFromTicks(cls.now - 60), 'IMSI901550000000082',
'192.168.99.2', 50, 80, 0, 0),
(psycopg2.TimestampFromTicks(cls.now - 10), 'IMSI901550000000082',
'192.168.99.2', 400, 300, 350, 220),
(psycopg2.TimestampFromTicks(cls.now - 5), 'IMSI901550000000082',
'192.168.99.2', 450, 325, 50, 25),
]
schema = ('record_timestamp, imsi, ipaddr, uploaded_bytes,'
' downloaded_bytes, uploaded_bytes_delta,'
' downloaded_bytes_delta')
connection = psycopg2.connect(host='localhost',
database='endaga',
user=PG_USER,
password=PG_PASSWORD)
with connection.cursor() as cursor:
for record in records:
values = "%s, '%s', '%s', %s, %s, %s, %s" % record
command = 'insert into gprs_records (%s) values(%s)' % (schema,
values)
cursor.execute(command)
connection.commit()
def gather_gprs_data():
"""Gets GPRS data from openbts-python and dumps it in the GPRS DB."""
gprs_db = gprs_database.GPRSDB()
try:
data = subscriber.get_gprs_usage()
except SubscriberNotFound:
return
for imsi in data.keys():
# If the IMSI is not a registered sub, ignore its data.
if not subscriber.get_subscribers(imsi=imsi):
continue
# Otherwise, get the IMSI's latest record and compute the byte count
# deltas.
record = gprs_db.get_latest_record(imsi)
if not record:
# No previous records exist for this IMSI so set the old byte
# counts to zero.
old_up_bytes = 0
old_down_bytes = 0
elif record['ipaddr'] != data[imsi]['ipaddr']:
# The ipaddr has been reset and with it, the byte count.
old_up_bytes = 0
old_down_bytes = 0
elif (record['uploaded_bytes'] > data[imsi]['uploaded_bytes']
or record['downloaded_bytes'] > data[imsi]['downloaded_bytes']):
# The ipaddr was recently re-assigned to this IMSI and it happens
# to match the IP we had previously. The byte count was reset
# during this transition.
old_up_bytes = 0
old_down_bytes = 0
else:
old_up_bytes = record['uploaded_bytes']
old_down_bytes = record['downloaded_bytes']
up_bytes_delta = data[imsi]['uploaded_bytes'] - old_up_bytes
down_bytes_delta = data[imsi]['downloaded_bytes'] - old_down_bytes
# Insert the GPRS data into the DB.
gprs_db.add_record(imsi, data[imsi]['ipaddr'],
data[imsi]['uploaded_bytes'],
data[imsi]['downloaded_bytes'], up_bytes_delta,
down_bytes_delta)