def chk_ipaddress(ipaddress): """ Check if IP address exists in our database """ DEV_ADD_IP = False # uncomment this to import from a fake different IP / used for dev # DEV_ADD_IP = '127.0.0.2' if DEV_ADD_IP: previous_ip = ipaddress ipaddress = DEV_ADD_IP try: switch = Switch.objects.get(ipaddress=ipaddress) except Switch.DoesNotExist: switch = Switch(name=ipaddress, ipaddress=ipaddress) switch.save() if not switch.id: print "Error when adding new Switch!" raise SystemExit if DEV_ADD_IP: ipaddress = previous_ip data = { 'ipaddress': ipaddress, 'switch': switch } return data
def chk_ipaddress(ipaddress): """ Check if IP address exists in our database """ DEV_ADD_IP = False # uncomment this to import from a fake different IP / used for dev # DEV_ADD_IP = '127.0.0.2' if DEV_ADD_IP: previous_ip = ipaddress ipaddress = DEV_ADD_IP try: switch = Switch.objects.get(ipaddress=ipaddress) except Switch.DoesNotExist: switch = Switch(name=ipaddress, ipaddress=ipaddress) switch.save() if not switch.id: print "Error when adding new Switch!" raise SystemExit if DEV_ADD_IP: ipaddress = previous_ip data = {'ipaddress': ipaddress, 'switch': switch} return data
def setUp(self): """Create model object""" self.user = User.objects.get(pk=1) # Switch model self.switch = Switch(name='localhost', ipaddress='127.0.0.1') self.switch.save() self.assertTrue(self.switch.__unicode__()) self.hangupcause = HangupCause(code=700, enumeration='UNALLOCATED_NUMBER') self.hangupcause.save() self.assertTrue(self.hangupcause.__unicode__())
def setUp(self): """Create model object""" self.user = User.objects.get(pk=1) # Switch model self.switch = Switch( name='localhost', ipaddress='127.0.0.1' ) self.switch.save() self.assertTrue(self.switch.__unicode__()) self.hangupcause = HangupCause( code=700, enumeration='UNALLOCATED_NUMBER' ) self.hangupcause.save() self.assertTrue(self.hangupcause.__unicode__())
def testSwitch(self): obj = Switch(name='localhost', ipaddress='127.0.0.1') obj.save() self.assertEquals('localhost', obj.name) self.assertNotEquals(obj.id, None) obj.delete()
class CdrModelTestCase(BaseAuthenticatedClient): """Test Switch, Alarm, HangupCause models""" fixtures = ['auth_user.json', 'hangup_cause.json'] def setUp(self): """Create model object""" self.user = User.objects.get(pk=1) # Switch model self.switch = Switch(name='localhost', ipaddress='127.0.0.1') self.switch.save() self.assertTrue(self.switch.__unicode__()) self.hangupcause = HangupCause(code=700, enumeration='UNALLOCATED_NUMBER') self.hangupcause.save() self.assertTrue(self.hangupcause.__unicode__()) def test_functions(self): get_switch_list() get_hangupcause_name(self.hangupcause.pk) get_hangupcause_name(2) get_hangupcause_id(self.hangupcause.code) # Template tags hangupcause_name_with_title(self.hangupcause.pk) value = {'_id': {'val': 1}} mongo_id(value, 'val') get_hc_list() get_country_id(['44', '442']) def test_cdr_search_form(self): data = { 'switch_id': 1, 'from_date': datetime.now().strftime("%Y-%m-%d"), 'to_date': datetime.now().strftime("%Y-%m-%d"), 'destination': 'abc', 'destination_type': 1, 'accountcode': 'abc', 'caller': 'abc', 'duration': 'abc', 'duration_type': '>', 'direction': 'INBOUND', 'hangup_cause': 1, 'result': 1, 'records_per_page': 10 } form = CdrSearchForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form["duration"].errors, ['abc is not a valid duration.']) #self.assertEqual(form["accountcode"].errors, ['abc is not a valid accountcode.']) def test_email_report_form(self): data = {'multiple_email': '[email protected],xyzlocalhost.com'} form = EmailReportForm(self.user, data) self.assertFalse(form.is_valid()) self.assertEqual(form["multiple_email"].errors, ['xyzlocalhost.com is not a valid e-mail address.']) data = {'multiple_email': '[email protected],[email protected]'} form = EmailReportForm(self.user, data) self.assertTrue(form.is_valid()) def test_model_value(self): """Create model object value""" self.assertEquals(self.switch.name, 'localhost') self.assertEquals(self.hangupcause.enumeration, 'UNALLOCATED_NUMBER') def tearDown(self): """Delete created object""" self.switch.delete() self.hangupcause.delete()
def import_cdr(shell=False): #TODO : dont use the args here # Browse settings.CDR_MONGO_IMPORT and for each IP check if the IP exist in our Switch objects # If it does we will connect to that Database and import the data as we do below print_shell(shell, "Starting the synchronization...") #loop within the Mongo CDR Import List for ipaddress in settings.CDR_MONGO_IMPORT: #Select the Switch ID print_shell(shell, "Switch : %s" % ipaddress) DEV_ADD_IP = False #uncomment this if you need to import from a fake different IP / used for dev #DEV_ADD_IP = '127.0.0.2' if DEV_ADD_IP: previous_ip = ipaddress ipaddress = DEV_ADD_IP try: switch = Switch.objects.get(ipaddress=ipaddress) except Switch.DoesNotExist: switch = Switch(name=ipaddress, ipaddress=ipaddress) switch.save() if not switch.id: print "Error when adding new Switch!" raise SystemExit if DEV_ADD_IP: ipaddress = previous_ip #Connect on MongoDB Database host = settings.CDR_MONGO_IMPORT[ipaddress]['host'] port = settings.CDR_MONGO_IMPORT[ipaddress]['port'] db_name = settings.CDR_MONGO_IMPORT[ipaddress]['db_name'] try: connection = Connection(host, port) DB_CONNECTION = connection[db_name] except ConnectionFailure, e: sys.stderr.write("Could not connect to MongoDB: %s - %s" % \ (e, ipaddress)) sys.exit(1) #Connect to Mongo importcdr_handler = DB_CONNECTION[settings.CDR_MONGO_IMPORT[ipaddress]['collection']] #total_record = importcdr_handler.find({'import_cdr': {'$exists': False}}).count() total_record = importcdr_handler.find({ '$or': [ {'import_cdr': {'$exists': False}}, {'import_cdr': 0} ] }).count() PAGE_SIZE = int(5000) total_loop_count = int( int(total_record) / PAGE_SIZE ) + 1 #print total_loop_count count_import = 0 for j in range(1, total_loop_count+1): PAGE_NUMBER = int(j) result = importcdr_handler.find({ '$or': [ {'import_cdr': {'$exists': False}}, {'import_cdr': 0} ] }, { "callflow.caller_profile.caller_id_number":1, "callflow.caller_profile.caller_id_name":1, "callflow.caller_profile.destination_number": 1, "variables.duration":1, "variables.billsec":1, "variables.hangup_cause_q850":1, "variables.accountcode":1, "variables.direction":1, "variables.uuid":1, "variables.remote_media_ip":1, "variables.start_uepoch":1, "variables.answer_uepoch":1, "variables.end_uepoch":1, "variables.mduration": 1, "variables.billmsec":1, "variables.read_codec": 1, "variables.write_codec": 1, "import_cdr_monthly": 1, "import_cdr_daily": 1, "import_cdr_hourly": 1, }).sort([ ('variables.start_uepoch', -1), ('callflow.caller_profile.destination_number', 1), ('variables.accountcode', 1), ('variables.hangup_cause_q850', 1) ]).limit(PAGE_SIZE) #Retrieve FreeSWITCH CDRs for cdr in result: start_uepoch = datetime.fromtimestamp(int(cdr['variables']['start_uepoch'][:10])) answer_uepoch = datetime.fromtimestamp(int(cdr['variables']['answer_uepoch'][:10])) end_uepoch = datetime.fromtimestamp(int(cdr['variables']['end_uepoch'][:10])) # Check Destination number destination_number = cdr['callflow']['caller_profile']['destination_number'] #remove prefix sanitized_destination = remove_prefix(destination_number, settings.PREFIX_TO_IGNORE) prefix_list = prefix_list_string(sanitized_destination) authorized = 1 # default #check desti against whiltelist authorized = chk_prefix_in_whitelist(prefix_list) if authorized: # allowed destination authorized = 1 else: #check against blacklist authorized = chk_prefix_in_blacklist(prefix_list) if not authorized: # not allowed destination authorized = 0 if len(sanitized_destination) < settings.PHONENUMBER_MIN_DIGITS: #It might be an extension country_id = 0 elif len(sanitized_destination) >= settings.PHONENUMBER_MIN_DIGITS \ and len(sanitized_destination) <= settings.PHONENUMBER_MAX_DIGITS: #It might be an local call print settings.LOCAL_DIALCODE #Need to add coma for get_country_id to eval correctly country_id = get_country_id(str(settings.LOCAL_DIALCODE) + ',') else: #International call country_id = get_country_id(prefix_list) if get_country_id==0: #TODO: Add logger print_shell(shell, "Error to find the country_id %s" % destination_number) destination_number = cdr['callflow']['caller_profile']['destination_number'] hangup_cause_id = get_hangupcause_id(cdr['variables']['hangup_cause_q850']) try: accountcode = cdr['variables']['accountcode'] except: accountcode = '' # Prepare global CDR cdr_record = { 'switch_id': switch.id, 'caller_id_number': cdr['callflow']['caller_profile']['caller_id_number'], 'caller_id_name': cdr['callflow']['caller_profile']['caller_id_name'], 'destination_number': destination_number, 'duration': int(cdr['variables']['duration']), 'billsec': int(cdr['variables']['billsec']), 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'direction': cdr['variables']['direction'], 'uuid': cdr['variables']['uuid'], 'remote_media_ip': cdr['variables']['remote_media_ip'], 'start_uepoch': start_uepoch, 'answer_uepoch': answer_uepoch, 'end_uepoch': end_uepoch, 'mduration': cdr['variables']['mduration'], 'billmsec': cdr['variables']['billmsec'], 'read_codec': cdr['variables']['read_codec'], 'write_codec': cdr['variables']['write_codec'], 'cdr_type': CDR_TYPE["freeswitch"], 'cdr_object_id': cdr['_id'], 'country_id': country_id, 'authorized': authorized, } # record global CDR CDR_COMMON.insert(cdr_record) print_shell(shell, "Sync CDR (cid:%s, dest:%s, dur:%s, hg:%s, country:%s, auth:%s)" % ( cdr['callflow']['caller_profile']['caller_id_number'], cdr['callflow']['caller_profile']['destination_number'], cdr['variables']['duration'], cdr['variables']['hangup_cause_q850'], country_id, authorized,)) count_import = count_import + 1 # change import_cdr flag #update_cdr_collection(importcdr_handler, cdr['_id'], 'import_cdr') # Store monthly cdr collection with unique import if not hasattr(cdr, 'import_cdr_monthly') or cdr['import_cdr_monthly'] == 0: # monthly collection current_y_m = datetime.strptime(str(start_uepoch)[:7], "%Y-%m") CDR_MONTHLY.update( { 'start_uepoch': current_y_m, 'destination_number': destination_number, 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'switch_id': switch.id, }, { '$inc': {'calls': 1, 'duration': int(cdr['variables']['duration']) } }, upsert=True) # Store daily cdr collection with unique import if not hasattr(cdr, 'import_cdr_daily') or cdr['import_cdr_daily'] == 0: # daily collection current_y_m_d = datetime.strptime(str(start_uepoch)[:10], "%Y-%m-%d") CDR_DAILY.update( { 'start_uepoch': current_y_m_d, 'destination_number': destination_number, 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'switch_id': switch.id, }, { '$inc': {'calls': 1, 'duration': int(cdr['variables']['duration']) } },upsert=True) # Store hourly cdr collection with unique import if not hasattr(cdr, 'import_cdr_hourly') or cdr['import_cdr_hourly'] == 0: # hourly collection current_y_m_d_h = datetime.strptime(str(start_uepoch)[:13], "%Y-%m-%d %H") CDR_HOURLY.update( { 'start_uepoch': current_y_m_d_h, 'destination_number': destination_number, 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'switch_id': switch.id,}, { '$inc': {'calls': 1, 'duration': int(cdr['variables']['duration']) } },upsert=True) # Country report collection current_y_m_d_h_m = datetime.strptime(str(start_uepoch)[:16], "%Y-%m-%d %H:%M") CDR_COUNTRY_REPORT.update( { 'start_uepoch': current_y_m_d_h_m, 'country_id': country_id, 'accountcode': accountcode, 'switch_id': switch.id,}, { '$inc': {'calls': 1, 'duration': int(cdr['variables']['duration']) } },upsert=True) # Flag the CDR as imported importcdr_handler.update( {'_id': cdr['_id']}, {'$set': {'import_cdr': 1, 'import_cdr_monthly': 1, 'import_cdr_daily': 1, 'import_cdr_hourly': 1}} ) if count_import > 0: # Apply index CDR_COMMON.ensure_index([("start_uepoch", -1)]) CDR_MONTHLY.ensure_index([("start_uepoch", -1)]) CDR_DAILY.ensure_index([("start_uepoch", -1)]) CDR_HOURLY.ensure_index([("start_uepoch", -1)]) CDR_COUNTRY_REPORT.ensure_index([("start_uepoch", -1)]) print_shell(shell, "Import on Switch(%s) - Record(s) imported:%d" % (ipaddress, count_import))
class CdrModelTestCase(BaseAuthenticatedClient): """Test Switch, Alarm, HangupCause models""" fixtures = ['auth_user.json', 'hangup_cause.json'] def setUp(self): """Create model object""" self.user = User.objects.get(pk=1) # Switch model self.switch = Switch( name='localhost', ipaddress='127.0.0.1' ) self.switch.save() self.assertTrue(self.switch.__unicode__()) self.hangupcause = HangupCause( code=700, enumeration='UNALLOCATED_NUMBER' ) self.hangupcause.save() self.assertTrue(self.hangupcause.__unicode__()) def test_functions(self): get_switch_list() get_hangupcause_name(self.hangupcause.pk) get_hangupcause_name(2) get_hangupcause_id(self.hangupcause.code) # Template tags hangupcause_name_with_title(self.hangupcause.pk) value = {'_id': {'val': 1}} mongo_id(value, 'val') get_hc_list() get_country_id(['44', '442']) def test_cdr_search_form(self): data = {'switch_id': 1, 'from_date': datetime.now().strftime("%Y-%m-%d"), 'to_date': datetime.now().strftime("%Y-%m-%d"), 'destination': 'abc', 'destination_type': 1, 'accountcode': 'abc', 'caller': 'abc', 'duration': 'abc', 'duration_type': '>', 'direction': 'INBOUND', 'hangup_cause': 1, 'result': 1, 'records_per_page': 10} form = CdrSearchForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form["duration"].errors, ['abc is not a valid duration.']) #self.assertEqual(form["accountcode"].errors, ['abc is not a valid accountcode.']) def test_email_report_form(self): data = {'multiple_email': '[email protected],xyzlocalhost.com'} form = EmailReportForm(self.user, data) self.assertFalse(form.is_valid()) self.assertEqual(form["multiple_email"].errors, ['xyzlocalhost.com is not a valid e-mail address.']) data = {'multiple_email': '[email protected],[email protected]'} form = EmailReportForm(self.user, data) self.assertTrue(form.is_valid()) def test_model_value(self): """Create model object value""" self.assertEquals(self.switch.name, 'localhost') self.assertEquals(self.hangupcause.enumeration, 'UNALLOCATED_NUMBER') def tearDown(self): """Delete created object""" self.switch.delete() self.hangupcause.delete()
def import_cdr_asterisk_mysql(shell=False): #TODO : dont use the args here # Browse settings.ASTERISK_CDR_MYSQL_IMPORT and for each IP check if the IP exist in our Switch objects # If it does we will connect to that Database and import the data as we do below print_shell(shell, "Starting the synchronization...") if settings.LOCAL_SWITCH_TYPE != 'asterisk': print_shell(shell, "The switch is not configured to import Asterisk...") return False #loop within the Mongo CDR Import List for ipaddress in settings.ASTERISK_CDR_MYSQL_IMPORT: #Select the Switch ID print_shell(shell, "Switch : %s" % ipaddress) DEV_ADD_IP = False #uncomment this if you need to import from a fake different IP / used for dev #DEV_ADD_IP = '127.0.0.2' if DEV_ADD_IP: previous_ip = ipaddress ipaddress = DEV_ADD_IP try: switch = Switch.objects.get(ipaddress=ipaddress) except Switch.DoesNotExist: switch = Switch(name=ipaddress, ipaddress=ipaddress) switch.save() if not switch.id: print_shell(shell, "Error when adding new Switch!") raise SystemExit if DEV_ADD_IP: ipaddress = previous_ip #Connect on Mysql Database db_name = settings.ASTERISK_CDR_MYSQL_IMPORT[ipaddress]['db_name'] table_name = settings.ASTERISK_CDR_MYSQL_IMPORT[ipaddress]['table_name'] user = settings.ASTERISK_CDR_MYSQL_IMPORT[ipaddress]['user'] password = settings.ASTERISK_CDR_MYSQL_IMPORT[ipaddress]['password'] host = settings.ASTERISK_CDR_MYSQL_IMPORT[ipaddress]['host'] try: connection = Database.connect(user=user, passwd=password, db=db_name, host=host) cursor = connection.cursor() cursor_update = connection.cursor() except: sys.stderr.write("Could not connect to Mysql: %s - %s" % \ (e, ipaddress)) sys.exit(1) try: cursor.execute("SELECT VERSION() from %s WHERE import_cdr IS NOT NULL LIMIT 0,1" % table_name) row = cursor.fetchone() except: #Add missing field to flag import cursor.execute("ALTER TABLE %s ADD import_cdr TINYINT NOT NULL DEFAULT '0'" % table_name) cursor.execute("ALTER TABLE %s ADD INDEX (import_cdr)" % table_name) #cursor.execute ("SELECT count(*) FROM %s WHERE import_cdr=0" % table_name) #row = cursor.fetchone() #total_record = row[0] #print total_loop_count count_import = 0 cursor.execute("SELECT dst, UNIX_TIMESTAMP(calldate), clid, channel, duration, billsec, disposition, accountcode, uniqueid, %s FROM %s WHERE import_cdr=0" % (settings.ASTERISK_PRIMARY_KEY, table_name)) row = cursor.fetchone() while row is not None: acctid = row[9] callerid = row[2] try: m = re.search('"(.+?)" <(.+?)>', callerid) callerid_name = m.group(1) callerid_number = m.group(2) except: callerid_name = '' callerid_number = callerid channel = row[3] try: duration = int(row[4]) except: duration = 0 try: billsec = int(row[5]) except: billsec = 0 ast_disposition = row[6] try: id_disposition = dic_disposition.get(ast_disposition.encode("utf-8"), 0) transdisposition = DISPOSITION_TRANSLATION[id_disposition] except: transdisposition = 0 hangup_cause_id = get_hangupcause_id(transdisposition) try: accountcode = int(row[7]) except: accountcode = '' uniqueid = row[8] start_uepoch = datetime.fromtimestamp(int(row[1])) answer_uepoch = start_uepoch end_uepoch = datetime.fromtimestamp(int(row[1]) + int(duration)) # Check Destination number destination_number = row[0] #TODO : Improve DRY duplicate code with import_cdr.py #remove prefix sanitized_destination = remove_prefix(destination_number, settings.PREFIX_TO_IGNORE) prefix_list = prefix_list_string(sanitized_destination) authorized = 1 # default #check desti against whiltelist authorized = chk_prefix_in_whitelist(prefix_list) if authorized: # allowed destination authorized = 1 else: #check against blacklist authorized = chk_prefix_in_blacklist(prefix_list) if not authorized: # not allowed destination authorized = 0 print sanitized_destination if len(sanitized_destination) < settings.PHONENUMBER_MIN_DIGITS: #It might be an extension country_id = 0 elif len(sanitized_destination) >= settings.PHONENUMBER_MIN_DIGITS \ and len(sanitized_destination) <= settings.PHONENUMBER_MAX_DIGITS: #It might be an local call print settings.LOCAL_DIALCODE #Need to add coma for get_country_id to eval correctly country_id = get_country_id(str(settings.LOCAL_DIALCODE) + ',') else: #International call country_id = get_country_id(prefix_list) if get_country_id==0: #TODO: Add logger print_shell(shell, "Error to find the country_id %s" % destination_number) # Prepare global CDR cdr_record = { 'switch_id': switch.id, 'caller_id_number': callerid_number, 'caller_id_name': callerid_name, 'destination_number': destination_number, 'duration': duration, 'billsec': billsec, 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'direction': "inbound", 'uuid': uniqueid, 'remote_media_ip': '', 'start_uepoch': start_uepoch, 'answer_uepoch': answer_uepoch, 'end_uepoch': end_uepoch, 'mduration': '', 'billmsec': '', 'read_codec': '', 'write_codec': '', 'cdr_type': CDR_TYPE["asterisk"], 'cdr_object_id': acctid, 'country_id': country_id, 'authorized': authorized, } # record global CDR CDR_COMMON.insert(cdr_record) print_shell(shell, "Sync CDR (%s:%d, cid:%s, dest:%s, dur:%s, hg:%s, country:%s, auth:%s, calldate:%s)" % ( settings.ASTERISK_PRIMARY_KEY, acctid, callerid_number, destination_number, duration, hangup_cause_id, country_id, authorized, start_uepoch.strftime('%Y-%m-%d %M:%S'),)) count_import = count_import + 1 # Store monthly cdr collection with unique import current_y_m = datetime.strptime(str(start_uepoch)[:7], "%Y-%m") CDR_MONTHLY.update( { 'start_uepoch': current_y_m, 'destination_number': destination_number, 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'switch_id': switch.id, }, { '$inc': {'calls': 1, 'duration': duration } }, upsert=True) # Store daily cdr collection with unique import current_y_m_d = datetime.strptime(str(start_uepoch)[:10], "%Y-%m-%d") CDR_DAILY.update( { 'start_uepoch': current_y_m_d, 'destination_number': destination_number, 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'switch_id': switch.id, }, { '$inc': {'calls': 1, 'duration': duration } },upsert=True) # Store hourly cdr collection with unique import current_y_m_d_h = datetime.strptime(str(start_uepoch)[:13], "%Y-%m-%d %H") CDR_HOURLY.update( { 'start_uepoch': current_y_m_d_h, 'destination_number': destination_number, 'hangup_cause_id': hangup_cause_id, 'accountcode': accountcode, 'switch_id': switch.id,}, { '$inc': {'calls': 1, 'duration': duration } },upsert=True) # Country report collection current_y_m_d_h_m = datetime.strptime(str(start_uepoch)[:16], "%Y-%m-%d %H:%M") CDR_COUNTRY_REPORT.update( { 'start_uepoch': current_y_m_d_h_m, 'country_id': country_id, 'accountcode': accountcode, 'switch_id': switch.id,}, { '$inc': {'calls': 1, 'duration': duration } },upsert=True) #Flag the CDR try: cursor_update.execute("UPDATE %s SET import_cdr=1 WHERE %s=%d" % (table_name, settings.ASTERISK_PRIMARY_KEY, acctid)) except: print_shell(shell, "ERROR : Update failed (%s:%d)" % (settings.ASTERISK_PRIMARY_KEY, acctid)) #Fetch a other record row = cursor.fetchone() cursor.close() cursor_update.close() connection.close() if count_import > 0: # Apply index CDR_COMMON.ensure_index([("start_uepoch", -1)]) CDR_MONTHLY.ensure_index([("start_uepoch", -1)]) CDR_DAILY.ensure_index([("start_uepoch", -1)]) CDR_HOURLY.ensure_index([("start_uepoch", -1)]) CDR_COUNTRY_REPORT.ensure_index([("start_uepoch", -1)]) print_shell(shell, "Import on Switch(%s) - Record(s) imported:%d" % (ipaddress, count_import))