def worker_thread(self, msg): large_package_msg = msg[4].split(';;') if len(large_package_msg) >= 3 and large_package_msg[0] == 'DATA': # Tengo datos para procesar dentro del mensaje largo #Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg client_pub_key_str_b64 = large_package_msg[1] client_pub_key_str = b64decode(large_package_msg[1]) client_signed_msg = b64decode(large_package_msg[2]) client_msg_filename = large_package_msg[3] client_plain_msg = b64decode(large_package_msg[4]) logger.info("Se ha recibido el siguiente paquete de datos: " + client_msg_filename) # print "<public key>\n" + client_pub_key_str + "\n</public key>\n" # print "Signed msg: " + client_signed_msg # print "Filename: " + client_msg_filename # print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n" # En el servidor se hace el VERIFY, para esto se necesita tambien la firma! pubKey = rsa.PublicKey.load_pkcs1(client_pub_key_str) if rsa.verify(client_plain_msg, client_signed_msg, pubKey): logger.debug("Chequeo de integridad satisfactorio para " + client_msg_filename) client_data = dbmanager.DBManager.getInstallationAndClientId( client_pub_key_str_b64) if client_data is not None: installation_id = client_data[0] client_id = client_data[1] client_ip = str(self.client_address[0]) #Client folder format: IP_cli_CLIENTID_ins_INSID client_server_folder = client_ip + "_cli_" + str( client_id) + "_ins_" + str(installation_id) logger.info("Salvando " + client_msg_filename + " en " + client_server_folder) client_records_server_folder = installDirUnix + "/records/" + client_server_folder if not os.path.exists(client_records_server_folder): logger.info("Creando directorio: " + client_server_folder) os.makedirs(client_records_server_folder) logFile = open( client_records_server_folder + "/" + client_msg_filename.split("/")[-1:][0], 'wb') logFile.write(client_plain_msg) logFile.close() # Check if there are old unusable files and remove them; we always need to keep only the REAL last hour of data remove_old_files(client_records_server_folder, client_msg_filename.split("/")[-1:][0]) # Check if we have at least 1hr (twelve 5 minutes files) of data if len(os.walk(client_records_server_folder).next() [2]) == 60: logger.info( "La instalacion " + client_server_folder + " tiene 1h de datos. Empezando procesamiento ...") # print "Starting calculation for the following files:" files_to_process = get_files_by_mdate( client_records_server_folder) # print files_to_process if len(files_to_process) < 60: logger.error("Error al procesar los archivos de " + client_server_folder) else: cwd = os.getcwd() os.chdir( '/home/pfitba/tix_production/data_processing') ansDictionary = completo_III.analyse_data( files_to_process) os.chdir(cwd) # Remove 10 oldest logs for count in range(0, 9): if os.path.isfile( files_to_process[count]) == True: os.remove(files_to_process[count]) try: new_isp_name = info.pais_num_name_nic( client_ip, 'EN')[1] logger.debug("ISP NAME = " + new_isp_name) except Exception, e: new_isp_name = 'Unknown' logger.debug("ISP not found for IP: " + str(client_ip)) payload = {'isp_name': str(new_isp_name)} headers = {'content-type': 'application/json'} r = requests.post(tixBaseUrl + 'bin/api/newISPPost', data=json.dumps(payload), headers=headers) jsonUserData = [] try: jsonUserData = json.loads( r.text ) # Parseo la respuesta JSON de la API de TiX except Exception, e: isp_id = 0 if (r is not None and len(jsonUserData) > 0): isp_id = jsonUserData['id'] else: logger.error( "No se ha podido insertar el nuevo ISP en la DB, se utilizara default (" + client_server_folder + ")") isp_id = 0 logger.debug( "Intentando insertar nuevo record en la DB de la carpeta: " + client_records_server_folder) try: dbmanager.DBManager.insert_record( ansDictionary['calidad_Down'], ansDictionary['utiliz_Down'], ansDictionary['H_RS_Down'], ansDictionary['H_Wave_Down'], time.strftime('%Y-%m-%d %H:%M:%S'), ansDictionary['calidad_Up'], ansDictionary['utiliz_Up'], ansDictionary['H_RS_Up'], ansDictionary['H_Wave_Up'], False, False, installation_id, isp_id, client_id) except Exception, e: logger.error( "Error al insertar nuevo record en la DB de la carpeta: " + client_records_server_folder) logger.error(e)
def handle(self): tstamp = ts() data = self.request[0].strip() socket = self.request[1] msg = data.split('!!') #este_thread=threading.current_thread() #threads_activos=threading.activeCount() #threading.enumerate() #print "{} wrote:".format(self.client_address[0]) # print msg[0]+'|'+msg[1]+'|'+msg[2]+'1'+msg[3] if len(msg ) > 4: # depende de si es un mensaje corto o un mensaje largo #Mensaje largo large_package_msg = msg[4].split(';;') if len(large_package_msg) >= 3 and large_package_msg[0] == 'DATA': # Tengo datos para procesar dentro del mensaje largo #Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg client_pub_key_str = b64decode(large_package_msg[1]) client_signed_msg = b64decode(large_package_msg[2]) client_msg_filename = large_package_msg[3] client_plain_msg = b64decode(large_package_msg[4]) print "Data package received!" print "<public key>\n" + client_pub_key_str + "\n</public key>\n" print "Signed msg: " + client_signed_msg print "Filename: " + client_msg_filename print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n" client_pub_key = RSA.importKey( client_pub_key_str) # import pub key from string # En el servidor se hace el VERIFY, para esto se necesita tambien la firma! signer = PKCS1_v1_5.new(client_pub_key) digest = SHA256.new() digest.update(client_plain_msg) if signer.verify(digest, client_signed_msg): print "Integrity check OK" client_data = dbmanager.DBManager.getInstallationAndClientId( client_pub_key_str) if client_data is not None: installation_id = client_data[0] client_id = client_data[1] #CLIENT FOLDER: IP_cli_CLIENTID_ins_INSID print "Saving data to: " + str( self.client_address[0]) + "_cli_" + str( client_id) + "_ins_" + str(installation_id) client_server_folder = str( self.client_address[0]) + "_cli_" + str( client_id) + "_ins_" + str(installation_id) print "Validando existencia de directorios para los records..." client_records_server_folder = installDirUnix + "/records/" + client_server_folder if not os.path.exists(client_records_server_folder): print "Creando directorio: " + client_server_folder os.makedirs(client_records_server_folder) logFile = open( client_records_server_folder + "/" + client_msg_filename, 'wb') logFile.write(client_plain_msg) logFile.close() # DBManager.insert_record(20,53,'2013-04-14 16:20:12.345678',55,50,"false","false",1,1,1) #downstream,downstreamcongestion,timestamp,upstream,upstreamcongestion,userdowncongestion,userupcongestion,installation_id,isp_id,user_id # Check if we have at least 1hr (six 10 minutes files) of data if len( os.walk(client_records_server_folder).next() [2]) >= 6: print "We have 1 hour data! Inserting new records in the DB ..." out_temp_filename = client_records_server_folder + '/last_records.temp' os.system('cat * > ' + out_temp_filename) # with open(outfilename, 'wb') as outfile: # for filename in glob.glob(client_records_server_folder + '/*'): # if(filename != out_temp_filename): # if(os.path.isfile(filename)): # with open(filename) as readfile: # shutil.copyfileobj(readfile, outfile) result = completo_III.analyse_data( client_records_server_folder + "/" + out_temp_filename) print result #TODO -> Remove # Remove tempfile if os.path.isfile(out_temp_filename) == True: os.remove(out_temp_filename) isp_id = 1 #TODO -> Find out dbmanager.DBManager.insert_record( randrange(100), randrange(100), 1, '2013-04-14 16:20:12.345678', randrange(100), randrange(100), 1, False, False, installation_id, isp_id, client_id) socket.sendto( msg[0] + '|' + tstamp + '|' + str(ts()) + '|' + msg[3] + '|' + msg[4], self.client_address) else: #Mensaje corto socket.sendto( msg[0] + '|' + tstamp + '|' + str(ts()) + '|' + msg[3], self.client_address)
print "Creando/validando Directorios para los records..." if not os.path.exists("/etc/TIX"): os.makedirs("/etc/TIX") os.makedirs("/etc/TIX/records") client_records_server_folder = "/etc/TIX/records/190.48.237.159_cli_1_ins_3" print len(os.walk(client_records_server_folder).next()) if len(os.walk(client_records_server_folder).next()[2]) >= 6: print "We have 1 hour data! Inserting new records in the DB ..." print "Starting calculation for the following files:" files_to_process = get_files_by_mdate(client_records_server_folder) print files_to_process if len(files_to_process) < 6: print "Error processing files; not enough files in directory" else: cwd = os.getcwd() os.chdir('/home/pfitba/ServerApp_25Nov/data_processing') print os.getcwd() result = completo_III.analyse_data(files_to_process) os.chdir(cwd) print result #TODO -> Remove # Remove oldest log if os.path.isfile(files_to_process[0]) == True: os.remove(files_to_process[0]) isp_id = 1 #TODO -> Find out
def handle(self): tstamp=ts() data = self.request[0].strip() socket = self.request[1] msg = data.split('!!') #este_thread=threading.current_thread() #threads_activos=threading.activeCount() #threading.enumerate() #print "{} wrote:".format(self.client_address[0]) # print msg[0]+'|'+msg[1]+'|'+msg[2]+'1'+msg[3] if len(msg)>4:# depende de si es un mensaje corto o un mensaje largo #Mensaje largo large_package_msg = msg[4].split(';;') if len(large_package_msg)>=3 and large_package_msg[0]=='DATA': # Tengo datos para procesar dentro del mensaje largo #Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg client_pub_key_str = b64decode(large_package_msg[1]) client_signed_msg = b64decode(large_package_msg[2]) client_msg_filename = large_package_msg[3] client_plain_msg = b64decode(large_package_msg[4]) print "Data package received!" print "<public key>\n" + client_pub_key_str + "\n</public key>\n" print "Signed msg: " + client_signed_msg print "Filename: " + client_msg_filename print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n" client_pub_key = RSA.importKey(client_pub_key_str) # import pub key from string # En el servidor se hace el VERIFY, para esto se necesita tambien la firma! signer = PKCS1_v1_5.new(client_pub_key) digest = SHA256.new() digest.update(client_plain_msg) if signer.verify(digest, client_signed_msg): print "Integrity check OK" client_data = dbmanager.DBManager.getInstallationAndClientId(client_pub_key_str) if client_data is not None: installation_id = client_data[0] client_id = client_data[1] #CLIENT FOLDER: IP_cli_CLIENTID_ins_INSID print "Saving data to: " + str(self.client_address[0]) + "_cli_" + str(client_id) + "_ins_" + str(installation_id) client_server_folder = str(self.client_address[0]) + "_cli_" + str(client_id) + "_ins_" + str(installation_id) print "Validando existencia de directorios para los records..." client_records_server_folder = installDirUnix + "/records/" + client_server_folder if not os.path.exists(client_records_server_folder): print "Creando directorio: " + client_server_folder os.makedirs(client_records_server_folder) logFile = open(client_records_server_folder + "/" + client_msg_filename, 'wb') logFile.write(client_plain_msg) logFile.close() # DBManager.insert_record(20,53,'2013-04-14 16:20:12.345678',55,50,"false","false",1,1,1) #downstream,downstreamcongestion,timestamp,upstream,upstreamcongestion,userdowncongestion,userupcongestion,installation_id,isp_id,user_id # Check if we have at least 1hr (six 10 minutes files) of data if len(os.walk(client_records_server_folder).next()[2]) >= 6: print "We have 1 hour data! Inserting new records in the DB ..." out_temp_filename = client_records_server_folder + '/last_records.temp' os.system('cat * > ' + out_temp_filename) # with open(outfilename, 'wb') as outfile: # for filename in glob.glob(client_records_server_folder + '/*'): # if(filename != out_temp_filename): # if(os.path.isfile(filename)): # with open(filename) as readfile: # shutil.copyfileobj(readfile, outfile) result = completo_III.analyse_data(client_records_server_folder + "/" + out_temp_filename) print result #TODO -> Remove # Remove tempfile if os.path.isfile(out_temp_filename) == True: os.remove(out_temp_filename) isp_id = 1 #TODO -> Find out dbmanager.DBManager.insert_record(randrange(100), randrange(100), 1, '2013-04-14 16:20:12.345678',randrange(100),randrange(100),1,False,False,installation_id,isp_id,client_id) socket.sendto(msg[0] + '|' + tstamp +'|' + str(ts()) + '|' + msg[3] + '|' + msg[4], self.client_address) else: #Mensaje corto socket.sendto(msg[0]+'|'+ tstamp +'|' + str(ts()) + '|' + msg[3], self.client_address)
def worker_thread(self, msg): large_package_msg = msg[4].split(';;') if len(large_package_msg)>=3 and large_package_msg[0]=='DATA': # Tengo datos para procesar dentro del mensaje largo #Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg client_pub_key_str_b64 = large_package_msg[1] client_pub_key_str = b64decode(large_package_msg[1]) client_signed_msg = b64decode(large_package_msg[2]) client_msg_filename = large_package_msg[3] client_plain_msg = b64decode(large_package_msg[4]) logger.info("Se ha recibido el siguiente paquete de datos: " + client_msg_filename) # print "<public key>\n" + client_pub_key_str + "\n</public key>\n" # print "Signed msg: " + client_signed_msg # print "Filename: " + client_msg_filename # print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n" # En el servidor se hace el VERIFY, para esto se necesita tambien la firma! pubKey = rsa.PublicKey.load_pkcs1(client_pub_key_str) if rsa.verify(client_plain_msg, client_signed_msg, pubKey): logger.debug("Chequeo de integridad satisfactorio para " + client_msg_filename) client_data = dbmanager.DBManager.getInstallationAndClientId(client_pub_key_str_b64) if client_data is not None: installation_id = client_data[0] client_id = client_data[1] client_ip = str(self.client_address[0]) #Client folder format: IP_cli_CLIENTID_ins_INSID client_server_folder = client_ip + "_cli_" + str(client_id) + "_ins_" + str(installation_id) logger.info("Salvando " + client_msg_filename + " en " + client_server_folder) client_records_server_folder = installDirUnix + "/records/" + client_server_folder if not os.path.exists(client_records_server_folder): logger.info("Creando directorio: " + client_server_folder) os.makedirs(client_records_server_folder) logFile = open(client_records_server_folder + "/" + client_msg_filename.split("/")[-1:][0], 'wb') logFile.write(client_plain_msg) logFile.close() # Check if there are old unusable files and remove them; we always need to keep only the REAL last hour of data remove_old_files(client_records_server_folder, client_msg_filename.split("/")[-1:][0]) # Check if we have at least 1hr (twelve 5 minutes files) of data if len(os.walk(client_records_server_folder).next()[2]) == 60: logger.info("La instalacion " + client_server_folder + " tiene 1h de datos. Empezando procesamiento ...") # print "Starting calculation for the following files:" files_to_process = get_files_by_mdate(client_records_server_folder) # print files_to_process if len(files_to_process) < 60: logger.error("Error al procesar los archivos de " + client_server_folder) else: cwd = os.getcwd() os.chdir('/home/pfitba/tix_production/data_processing') ansDictionary = completo_III.analyse_data(files_to_process) os.chdir(cwd) # Remove 10 oldest logs for count in range(0,9): if os.path.isfile(files_to_process[count]) == True: os.remove(files_to_process[count]) try: new_isp_name = info.pais_num_name_nic(client_ip, 'EN' )[1] logger.debug("ISP NAME = " + new_isp_name) except Exception, e: new_isp_name = 'Unknown' logger.debug("ISP not found for IP: " + str(client_ip)) payload = {'isp_name': str(new_isp_name)} headers = {'content-type': 'application/json'} r = requests.post(tixBaseUrl + 'bin/api/newISPPost', data=json.dumps(payload), headers=headers) jsonUserData = [] try: jsonUserData = json.loads(r.text) # Parseo la respuesta JSON de la API de TiX except Exception, e: isp_id = 0 if(r is not None and len(jsonUserData) > 0): isp_id = jsonUserData['id'] else: logger.error("No se ha podido insertar el nuevo ISP en la DB, se utilizara default (" + client_server_folder + ")") isp_id = 0 logger.debug("Intentando insertar nuevo record en la DB de la carpeta: " + client_records_server_folder) try: dbmanager.DBManager.insert_record(ansDictionary['calidad_Down'],ansDictionary['utiliz_Down'],ansDictionary['H_RS_Down'],ansDictionary['H_Wave_Down'],time.strftime('%Y-%m-%d %H:%M:%S'),ansDictionary['calidad_Up'],ansDictionary['utiliz_Up'],ansDictionary['H_RS_Up'],ansDictionary['H_Wave_Up'],False,False,installation_id,isp_id,client_id) except Exception, e: logger.error("Error al insertar nuevo record en la DB de la carpeta: " + client_records_server_folder) logger.error(e)