def worker_thread(self, msg):
large_package_msg = msg[4].split(';;')
if len(large_package_msg) >= 3 and large_package_msg[0] == 'DATA':
# Tengo datos para procesar dentro del mensaje largo
#Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg
client_pub_key_str_b64 = large_package_msg[1]
client_pub_key_str = b64decode(large_package_msg[1])
client_signed_msg = b64decode(large_package_msg[2])
client_msg_filename = large_package_msg[3]
client_plain_msg = b64decode(large_package_msg[4])
logger.info("Se ha recibido el siguiente paquete de datos: " +
client_msg_filename)
# print "<public key>\n" + client_pub_key_str + "\n</public key>\n"
# print "Signed msg: " + client_signed_msg
# print "Filename: " + client_msg_filename
# print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n"
# En el servidor se hace el VERIFY, para esto se necesita tambien la firma!
pubKey = rsa.PublicKey.load_pkcs1(client_pub_key_str)
if rsa.verify(client_plain_msg, client_signed_msg, pubKey):
logger.debug("Chequeo de integridad satisfactorio para " +
client_msg_filename)
client_data = dbmanager.DBManager.getInstallationAndClientId(
client_pub_key_str_b64)
if client_data is not None:
installation_id = client_data[0]
client_id = client_data[1]
client_ip = str(self.client_address[0])
#Client folder format: IP_cli_CLIENTID_ins_INSID
client_server_folder = client_ip + "_cli_" + str(
client_id) + "_ins_" + str(installation_id)
logger.info("Salvando " + client_msg_filename + " en " +
client_server_folder)
client_records_server_folder = installDirUnix + "/records/" + client_server_folder
if not os.path.exists(client_records_server_folder):
logger.info("Creando directorio: " +
client_server_folder)
os.makedirs(client_records_server_folder)
logFile = open(
client_records_server_folder + "/" +
client_msg_filename.split("/")[-1:][0], 'wb')
logFile.write(client_plain_msg)
logFile.close()
# Check if there are old unusable files and remove them; we always need to keep only the REAL last hour of data
remove_old_files(client_records_server_folder,
client_msg_filename.split("/")[-1:][0])
# Check if we have at least 1hr (twelve 5 minutes files) of data
if len(os.walk(client_records_server_folder).next()
[2]) == 60:
logger.info(
"La instalacion " + client_server_folder +
" tiene 1h de datos. Empezando procesamiento ...")
# print "Starting calculation for the following files:"
files_to_process = get_files_by_mdate(
client_records_server_folder)
# print files_to_process
if len(files_to_process) < 60:
logger.error("Error al procesar los archivos de " +
client_server_folder)
else:
cwd = os.getcwd()
os.chdir(
'/home/pfitba/tix_production/data_processing')
ansDictionary = completo_III.analyse_data(
files_to_process)
os.chdir(cwd)
# Remove 10 oldest logs
for count in range(0, 9):
if os.path.isfile(
files_to_process[count]) == True:
os.remove(files_to_process[count])
try:
new_isp_name = info.pais_num_name_nic(
client_ip, 'EN')[1]
logger.debug("ISP NAME = " + new_isp_name)
except Exception, e:
new_isp_name = 'Unknown'
logger.debug("ISP not found for IP: " +
str(client_ip))
payload = {'isp_name': str(new_isp_name)}
headers = {'content-type': 'application/json'}
r = requests.post(tixBaseUrl +
'bin/api/newISPPost',
data=json.dumps(payload),
headers=headers)
jsonUserData = []
try:
jsonUserData = json.loads(
r.text
) # Parseo la respuesta JSON de la API de TiX
except Exception, e:
isp_id = 0
if (r is not None and len(jsonUserData) > 0):
isp_id = jsonUserData['id']
else:
logger.error(
"No se ha podido insertar el nuevo ISP en la DB, se utilizara default ("
+ client_server_folder + ")")
isp_id = 0
logger.debug(
"Intentando insertar nuevo record en la DB de la carpeta: "
+ client_records_server_folder)
try:
dbmanager.DBManager.insert_record(
ansDictionary['calidad_Down'],
ansDictionary['utiliz_Down'],
ansDictionary['H_RS_Down'],
ansDictionary['H_Wave_Down'],
time.strftime('%Y-%m-%d %H:%M:%S'),
ansDictionary['calidad_Up'],
ansDictionary['utiliz_Up'],
ansDictionary['H_RS_Up'],
ansDictionary['H_Wave_Up'], False, False,
installation_id, isp_id, client_id)
except Exception, e:
logger.error(
"Error al insertar nuevo record en la DB de la carpeta: "
+ client_records_server_folder)
logger.error(e)
def handle(self):
tstamp = ts()
data = self.request[0].strip()
socket = self.request[1]
msg = data.split('!!')
#este_thread=threading.current_thread()
#threads_activos=threading.activeCount() #threading.enumerate()
#print "{} wrote:".format(self.client_address[0])
# print msg[0]+'|'+msg[1]+'|'+msg[2]+'1'+msg[3]
if len(msg
) > 4: # depende de si es un mensaje corto o un mensaje largo
#Mensaje largo
large_package_msg = msg[4].split(';;')
if len(large_package_msg) >= 3 and large_package_msg[0] == 'DATA':
# Tengo datos para procesar dentro del mensaje largo
#Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg
client_pub_key_str = b64decode(large_package_msg[1])
client_signed_msg = b64decode(large_package_msg[2])
client_msg_filename = large_package_msg[3]
client_plain_msg = b64decode(large_package_msg[4])
print "Data package received!"
print "<public key>\n" + client_pub_key_str + "\n</public key>\n"
print "Signed msg: " + client_signed_msg
print "Filename: " + client_msg_filename
print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n"
client_pub_key = RSA.importKey(
client_pub_key_str) # import pub key from string
# En el servidor se hace el VERIFY, para esto se necesita tambien la firma!
signer = PKCS1_v1_5.new(client_pub_key)
digest = SHA256.new()
digest.update(client_plain_msg)
if signer.verify(digest, client_signed_msg):
print "Integrity check OK"
client_data = dbmanager.DBManager.getInstallationAndClientId(
client_pub_key_str)
if client_data is not None:
installation_id = client_data[0]
client_id = client_data[1]
#CLIENT FOLDER: IP_cli_CLIENTID_ins_INSID
print "Saving data to: " + str(
self.client_address[0]) + "_cli_" + str(
client_id) + "_ins_" + str(installation_id)
client_server_folder = str(
self.client_address[0]) + "_cli_" + str(
client_id) + "_ins_" + str(installation_id)
print "Validando existencia de directorios para los records..."
client_records_server_folder = installDirUnix + "/records/" + client_server_folder
if not os.path.exists(client_records_server_folder):
print "Creando directorio: " + client_server_folder
os.makedirs(client_records_server_folder)
logFile = open(
client_records_server_folder + "/" +
client_msg_filename, 'wb')
logFile.write(client_plain_msg)
logFile.close()
# DBManager.insert_record(20,53,'2013-04-14 16:20:12.345678',55,50,"false","false",1,1,1)
#downstream,downstreamcongestion,timestamp,upstream,upstreamcongestion,userdowncongestion,userupcongestion,installation_id,isp_id,user_id
# Check if we have at least 1hr (six 10 minutes files) of data
if len(
os.walk(client_records_server_folder).next()
[2]) >= 6:
print "We have 1 hour data! Inserting new records in the DB ..."
out_temp_filename = client_records_server_folder + '/last_records.temp'
os.system('cat * > ' + out_temp_filename)
# with open(outfilename, 'wb') as outfile:
# for filename in glob.glob(client_records_server_folder + '/*'):
# if(filename != out_temp_filename):
# if(os.path.isfile(filename)):
# with open(filename) as readfile:
# shutil.copyfileobj(readfile, outfile)
result = completo_III.analyse_data(
client_records_server_folder + "/" +
out_temp_filename)
print result #TODO -> Remove
# Remove tempfile
if os.path.isfile(out_temp_filename) == True:
os.remove(out_temp_filename)
isp_id = 1 #TODO -> Find out
dbmanager.DBManager.insert_record(
randrange(100), randrange(100), 1,
'2013-04-14 16:20:12.345678', randrange(100),
randrange(100), 1, False, False,
installation_id, isp_id, client_id)
socket.sendto(
msg[0] + '|' + tstamp + '|' + str(ts()) + '|' + msg[3] + '|' +
msg[4], self.client_address)
else:
#Mensaje corto
socket.sendto(
msg[0] + '|' + tstamp + '|' + str(ts()) + '|' + msg[3],
self.client_address)
print "Creando/validando Directorios para los records..."
if not os.path.exists("/etc/TIX"):
os.makedirs("/etc/TIX")
os.makedirs("/etc/TIX/records")
client_records_server_folder = "/etc/TIX/records/190.48.237.159_cli_1_ins_3"
print len(os.walk(client_records_server_folder).next())
if len(os.walk(client_records_server_folder).next()[2]) >= 6:
print "We have 1 hour data! Inserting new records in the DB ..."
print "Starting calculation for the following files:"
files_to_process = get_files_by_mdate(client_records_server_folder)
print files_to_process
if len(files_to_process) < 6:
print "Error processing files; not enough files in directory"
else:
cwd = os.getcwd()
os.chdir('/home/pfitba/ServerApp_25Nov/data_processing')
print os.getcwd()
result = completo_III.analyse_data(files_to_process)
os.chdir(cwd)
print result #TODO -> Remove
# Remove oldest log
if os.path.isfile(files_to_process[0]) == True:
os.remove(files_to_process[0])
isp_id = 1 #TODO -> Find out
def handle(self):
tstamp=ts()
data = self.request[0].strip()
socket = self.request[1]
msg = data.split('!!')
#este_thread=threading.current_thread()
#threads_activos=threading.activeCount() #threading.enumerate()
#print "{} wrote:".format(self.client_address[0])
# print msg[0]+'|'+msg[1]+'|'+msg[2]+'1'+msg[3]
if len(msg)>4:# depende de si es un mensaje corto o un mensaje largo
#Mensaje largo
large_package_msg = msg[4].split(';;')
if len(large_package_msg)>=3 and large_package_msg[0]=='DATA':
# Tengo datos para procesar dentro del mensaje largo
#Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg
client_pub_key_str = b64decode(large_package_msg[1])
client_signed_msg = b64decode(large_package_msg[2])
client_msg_filename = large_package_msg[3]
client_plain_msg = b64decode(large_package_msg[4])
print "Data package received!"
print "<public key>\n" + client_pub_key_str + "\n</public key>\n"
print "Signed msg: " + client_signed_msg
print "Filename: " + client_msg_filename
print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n"
client_pub_key = RSA.importKey(client_pub_key_str) # import pub key from string
# En el servidor se hace el VERIFY, para esto se necesita tambien la firma!
signer = PKCS1_v1_5.new(client_pub_key)
digest = SHA256.new()
digest.update(client_plain_msg)
if signer.verify(digest, client_signed_msg):
print "Integrity check OK"
client_data = dbmanager.DBManager.getInstallationAndClientId(client_pub_key_str)
if client_data is not None:
installation_id = client_data[0]
client_id = client_data[1]
#CLIENT FOLDER: IP_cli_CLIENTID_ins_INSID
print "Saving data to: " + str(self.client_address[0]) + "_cli_" + str(client_id) + "_ins_" + str(installation_id)
client_server_folder = str(self.client_address[0]) + "_cli_" + str(client_id) + "_ins_" + str(installation_id)
print "Validando existencia de directorios para los records..."
client_records_server_folder = installDirUnix + "/records/" + client_server_folder
if not os.path.exists(client_records_server_folder):
print "Creando directorio: " + client_server_folder
os.makedirs(client_records_server_folder)
logFile = open(client_records_server_folder + "/" + client_msg_filename, 'wb')
logFile.write(client_plain_msg)
logFile.close()
# DBManager.insert_record(20,53,'2013-04-14 16:20:12.345678',55,50,"false","false",1,1,1)
#downstream,downstreamcongestion,timestamp,upstream,upstreamcongestion,userdowncongestion,userupcongestion,installation_id,isp_id,user_id
# Check if we have at least 1hr (six 10 minutes files) of data
if len(os.walk(client_records_server_folder).next()[2]) >= 6:
print "We have 1 hour data! Inserting new records in the DB ..."
out_temp_filename = client_records_server_folder + '/last_records.temp'
os.system('cat * > ' + out_temp_filename)
# with open(outfilename, 'wb') as outfile:
# for filename in glob.glob(client_records_server_folder + '/*'):
# if(filename != out_temp_filename):
# if(os.path.isfile(filename)):
# with open(filename) as readfile:
# shutil.copyfileobj(readfile, outfile)
result = completo_III.analyse_data(client_records_server_folder + "/" + out_temp_filename)
print result #TODO -> Remove
# Remove tempfile
if os.path.isfile(out_temp_filename) == True:
os.remove(out_temp_filename)
isp_id = 1 #TODO -> Find out
dbmanager.DBManager.insert_record(randrange(100), randrange(100), 1, '2013-04-14 16:20:12.345678',randrange(100),randrange(100),1,False,False,installation_id,isp_id,client_id)
socket.sendto(msg[0] + '|' + tstamp +'|' + str(ts()) + '|' + msg[3] + '|' + msg[4], self.client_address)
else:
#Mensaje corto
socket.sendto(msg[0]+'|'+ tstamp +'|' + str(ts()) + '|' + msg[3], self.client_address)
print "Creando/validando Directorios para los records..."
if not os.path.exists("/etc/TIX"):
os.makedirs("/etc/TIX")
os.makedirs("/etc/TIX/records")
client_records_server_folder = "/etc/TIX/records/190.48.237.159_cli_1_ins_3"
print len(os.walk(client_records_server_folder).next())
if len(os.walk(client_records_server_folder).next()[2]) >= 6:
print "We have 1 hour data! Inserting new records in the DB ..."
print "Starting calculation for the following files:"
files_to_process = get_files_by_mdate(client_records_server_folder)
print files_to_process
if len(files_to_process) < 6:
print "Error processing files; not enough files in directory"
else:
cwd = os.getcwd()
os.chdir('/home/pfitba/ServerApp_25Nov/data_processing')
print os.getcwd()
result = completo_III.analyse_data(files_to_process)
os.chdir(cwd)
print result #TODO -> Remove
# Remove oldest log
if os.path.isfile(files_to_process[0]) == True:
os.remove(files_to_process[0])
isp_id = 1 #TODO -> Find out
def worker_thread(self, msg):
large_package_msg = msg[4].split(';;')
if len(large_package_msg)>=3 and large_package_msg[0]=='DATA':
# Tengo datos para procesar dentro del mensaje largo
#Cliente envia al server: DATA|publicKeyPlain|signedMeessage|msg
client_pub_key_str_b64 = large_package_msg[1]
client_pub_key_str = b64decode(large_package_msg[1])
client_signed_msg = b64decode(large_package_msg[2])
client_msg_filename = large_package_msg[3]
client_plain_msg = b64decode(large_package_msg[4])
logger.info("Se ha recibido el siguiente paquete de datos: " + client_msg_filename)
# print "<public key>\n" + client_pub_key_str + "\n</public key>\n"
# print "Signed msg: " + client_signed_msg
# print "Filename: " + client_msg_filename
# print "<plain msg>\n" + client_plain_msg + "\n</plain msg>\n"
# En el servidor se hace el VERIFY, para esto se necesita tambien la firma!
pubKey = rsa.PublicKey.load_pkcs1(client_pub_key_str)
if rsa.verify(client_plain_msg, client_signed_msg, pubKey):
logger.debug("Chequeo de integridad satisfactorio para " + client_msg_filename)
client_data = dbmanager.DBManager.getInstallationAndClientId(client_pub_key_str_b64)
if client_data is not None:
installation_id = client_data[0]
client_id = client_data[1]
client_ip = str(self.client_address[0])
#Client folder format: IP_cli_CLIENTID_ins_INSID
client_server_folder = client_ip + "_cli_" + str(client_id) + "_ins_" + str(installation_id)
logger.info("Salvando " + client_msg_filename + " en " + client_server_folder)
client_records_server_folder = installDirUnix + "/records/" + client_server_folder
if not os.path.exists(client_records_server_folder):
logger.info("Creando directorio: " + client_server_folder)
os.makedirs(client_records_server_folder)
logFile = open(client_records_server_folder + "/" + client_msg_filename.split("/")[-1:][0], 'wb')
logFile.write(client_plain_msg)
logFile.close()
# Check if there are old unusable files and remove them; we always need to keep only the REAL last hour of data
remove_old_files(client_records_server_folder, client_msg_filename.split("/")[-1:][0])
# Check if we have at least 1hr (twelve 5 minutes files) of data
if len(os.walk(client_records_server_folder).next()[2]) == 60:
logger.info("La instalacion " + client_server_folder + " tiene 1h de datos. Empezando procesamiento ...")
# print "Starting calculation for the following files:"
files_to_process = get_files_by_mdate(client_records_server_folder)
# print files_to_process
if len(files_to_process) < 60:
logger.error("Error al procesar los archivos de " + client_server_folder)
else:
cwd = os.getcwd()
os.chdir('/home/pfitba/tix_production/data_processing')
ansDictionary = completo_III.analyse_data(files_to_process)
os.chdir(cwd)
# Remove 10 oldest logs
for count in range(0,9):
if os.path.isfile(files_to_process[count]) == True:
os.remove(files_to_process[count])
try:
new_isp_name = info.pais_num_name_nic(client_ip, 'EN' )[1]
logger.debug("ISP NAME = " + new_isp_name)
except Exception, e:
new_isp_name = 'Unknown'
logger.debug("ISP not found for IP: " + str(client_ip))
payload = {'isp_name': str(new_isp_name)}
headers = {'content-type': 'application/json'}
r = requests.post(tixBaseUrl + 'bin/api/newISPPost', data=json.dumps(payload), headers=headers)
jsonUserData = []
try:
jsonUserData = json.loads(r.text) # Parseo la respuesta JSON de la API de TiX
except Exception, e:
isp_id = 0
if(r is not None and len(jsonUserData) > 0):
isp_id = jsonUserData['id']
else:
logger.error("No se ha podido insertar el nuevo ISP en la DB, se utilizara default (" + client_server_folder + ")")
isp_id = 0
logger.debug("Intentando insertar nuevo record en la DB de la carpeta: " + client_records_server_folder)
try:
dbmanager.DBManager.insert_record(ansDictionary['calidad_Down'],ansDictionary['utiliz_Down'],ansDictionary['H_RS_Down'],ansDictionary['H_Wave_Down'],time.strftime('%Y-%m-%d %H:%M:%S'),ansDictionary['calidad_Up'],ansDictionary['utiliz_Up'],ansDictionary['H_RS_Up'],ansDictionary['H_Wave_Up'],False,False,installation_id,isp_id,client_id)
except Exception, e:
logger.error("Error al insertar nuevo record en la DB de la carpeta: " + client_records_server_folder)
logger.error(e)
