def get(self):
"""Recupera lista de todos os modelos disponíveis na pasta da aplicação (genereated e uploaded)"""
try:
logger.info("Inicio do metodo [ModelList] [GET]....")
files_models = list()
if os.path.exists(path_app_model_uploaded):
files_uploaded = [
file_json
for file_json in os.listdir(path_app_model_uploaded)
if (file_json.endswith('.joblib'))
]
for file_upload in files_uploaded:
files_models.append(name_folder_uploaded + "/" +
file_upload)
if os.path.exists(path_app_model_generated):
files_generated = [
file_json
for file_json in os.listdir(path_app_model_generated)
if (file_json.endswith('.joblib'))
]
for file_generated in files_generated:
files_models.append(name_folder_generated + "/" +
file_generated)
logger.info("Quantidade de arquivos: " + str(len(files_models)))
return {'is_ok': 'true', 'result': str(files_models)}
except Exception as err:
logger.error(traceback.format_exc())
msg_err = str(err).replace("'", '"')
return {'is_ok': 'false', 'result': str(msg_err)}
def post(self):
try:
logger.info("Requisição [Auth] [POST] iniciada...")
result_exec = exec_auth()
return jsonify({'is_ok': 'true', 'result': result_exec})
except Exception as ex:
logger.error(traceback.format_exc())
msg_err = str(ex).replace("'", '"')
return {'is_ok': 'false', 'result': str(msg_err)}, 401
def post(self):
try:
logger.info("Inicio do metodo [ModelUpload] [POST]....")
service_model = ModelService()
service_model.upload_model(request)
return {'is_ok': 'true', 'result': 'Upload Realizado com sucesso.'}
except Exception as err:
logger.error(traceback.format_exc())
msg_err = str(err).replace("'", '"')
return {'is_ok': 'false', 'result': str(msg_err)}
def get(self):
logger.info("Requisição [LogActual] [GET] iniciada...")
try:
service = LogService()
result_exec = service.read_log_atual('app.log')
return {'is_ok': 'true', 'result': result_exec}
except Exception as ex:
logger.error(traceback.format_exc())
msg_err = str(ex).replace("'", '"')
return {'is_ok': 'false', 'result': str(msg_err)}
def read_log_atual(self, nome_arquivo_log):
logger.info("Inicio do metodo [read_log_atual]")
if not os.path.exists(path_app_log):
raise Exception("Diretório " + path_app_log + " não existe.")
if not os.path.isfile(path_app_log + "/" + nome_arquivo_log):
raise Exception("File " + nome_arquivo_log + " not found.")
with open(path_app_log + '/' + nome_arquivo_log,
'r',
encoding='ISO-8859-1') as file:
return file.read().splitlines()
def post(self):
try:
logger.info("Inicio do metodo [Model] [POST]....")
service = ModelService()
result_exec = service.execute_model(request)
return jsonify({
'is_ok': 'true',
'result': result_exec.to_dict('index')
})
except Exception as err:
logger.error(traceback.format_exc())
msg_err = str(err).replace("'", '"')
return {'is_ok': 'false', 'result': str(msg_err)}
def save_file_model_from_request(request_received,
field_request=None,
path_folder_upload=None):
logger.info("Inicio do método [salvar_arquivo_modelo_from_request]")
if path_folder_upload is None:
raise Exception("Parâmetro [path_folder_upload] é None.")
if field_request is None:
raise Exception("Parâmetro [field_request] é None.")
logger.info("===> " + str(request_received.files))
if field_request not in request_received.files:
raise Exception("O campo [" + field_request +
"] NÃO está presente na requisição.....")
file = request_received.files[field_request]
if file.filename == '' or file.filename is None:
raise Exception("Nenhum arquivo foi enviado na requisição...")
filename = secure_filename(file.filename)
arquivo_modelo = os.path.join(path_folder_upload, filename)
if os.path.isfile(arquivo_modelo):
raise Exception("Arquivo já existe na pasta de uploaded...")
file.save(arquivo_modelo)
file_name = file.filename.rsplit('.', 1)[0]
file_type = file.filename.rsplit('.', 1)[1]
logger.info(" Nome Arquivo: [" + file_name + "]")
logger.info("Extensão Arquivo: [" + file_type + "]")
return file.filename
def deploy(self, filename):
logger.info("Inicio do metodo [deploy]")
if not os.path.exists(
path=configuration.configuration_app.path_file_symbol):
logger.info("Criando diretório [" +
configuration.configuration_app.path_file_symbol +
"] pois o mesmo não existe...")
os.mkdir(configuration.configuration_app.path_file_symbol)
symbol_link = configuration.configuration_app.path_file_symbol + "/" + configuration.configuration_app.name_file_symbol
file_target = path_app_model + "/" + filename
if not os.path.exists(file_target):
raise Exception("Arquivo informado não foi encontrado.")
create_syslink(path_syslink=symbol_link, path_file_target=file_target)
def create_syslink(path_syslink, path_file_target):
logger.info("Inicio do metodo [deploy_model_generated]")
if os.path.isfile(path_syslink):
logger.info("Excluindo arquivo [" + path_syslink + "]")
os.remove(path_syslink)
logger.info("SYSLINK a ser criado [" + path_syslink + "]")
logger.info("Criando SYSLINK do arquivo [" + path_file_target + "]")
os.symlink(path_file_target, path_syslink)
def memory_monitor():
while True:
memory_percent = psutil.virtual_memory().percent
ram_total = int(int(psutil.virtual_memory().total) / 1024 / 1024)
memory_swap = int(int(psutil.swap_memory().used) / 1024 / 1024)
memory_swap_total = int(int(psutil.swap_memory().total) / 1024 / 1024)
memory_swap_percent = psutil.swap_memory().percent
logger.info("===> RAM total is {} MB ### "
"RAM usage is {} % ### "
"Swap total is {} MB ### "
"Swap usage is {} MB ### "
"Swap usage is {} %".format(ram_total, memory_percent,
memory_swap, memory_swap_total,
memory_swap_percent))
time.sleep(configuration.configuration_app.refresh_time_monitor)
def post(self):
"""Implanta um determinado modelo para que seja utilizado pela aplicação"""
try:
logger.info("Inicio do metodo [ModelDeploy] [POST]....")
service = ModelService()
data = json.loads(request.data)
filename = data.get("filename", None)
if filename is None:
return {
"result":
"O campo filename não foi encontrado no corpo da requisição"
}
service.deploy(filename)
return {'is_ok': 'true', 'result': 'Deploy realizado com sucesso.'}
except Exception as err:
logger.error(traceback.format_exc())
msg_err = str(err).replace("'", '"')
return {'is_ok': 'false', 'result': str(msg_err)}
def move_file_datetime(path_aquivo, formato_data, formato_hora, path_destino):
logger.info("")
logger.info("Inicio do Metodo [mover_arquivo_com_data_hora]")
logger.info("Movendo arquivo [" + path_aquivo + "] para pasta [" +
path_destino + "] ...")
data_atual = datetime.now()
data_file_discard = data_atual.strftime(
formato_data) + "_" + data_atual.strftime(formato_hora + "_")
ultima_posicao_barra = path_aquivo.rfind("/")
novo_nome_arquivo = path_destino + "/" + data_file_discard + path_aquivo[
ultima_posicao_barra + 1:]
logger.info("Novo nome do arquivo [" + novo_nome_arquivo + "]")
os.rename(path_aquivo, novo_nome_arquivo)
def get(self):
try:
logger.info("Inicio do metodo [ModelFile] [GET]....")
if not os.path.exists(path_app_model):
raise Exception("Diretório " + path_app_model + " não existe.")
if not os.path.isfile(
configuration.configuration_app.path_file_symbol + "/" +
configuration.configuration_app.name_file_symbol):
raise Exception(
"Arquivo [" +
configuration.configuration_app.path_file_symbol + "/" +
configuration.configuration_app.name_file_symbol +
" não encontrado")
return send_from_directory(
directory=configuration.configuration_app.path_file_symbol,
filename=configuration.configuration_app.name_file_symbol,
as_attachment=True)
except Exception as err:
logger.error(traceback.format_exc())
msg_err = str(err).replace("'", '"')
return {'is_ok': 'false', 'result': str(msg_err)}
def exec_auth():
logger.info("Inicio do metodo [auth]")
auth = request.authorization
if not auth or not auth.username or not auth.password:
raise Exception("WWW-Authenticate': 'Basic auth=Login required")
logger.info("====> Password Hash: " +
generate_password_hash(auth.password))
if auth.username != configuration.configuration_app.username:
raise Exception("Usuário incorreto")
if check_password_hash(configuration.configuration_app.password,
auth.password):
exp_time = datetime.datetime.now() + datetime.timedelta(hours=12)
token = jwt.encode(
{
'username': configuration.configuration_app.username,
'exp': exp_time
}, configuration.configuration_app.token_secret)
return {
'message': 'Login realizado com sucesso',
'token': token,
'exp': exp_time
}
else:
raise Exception("Senha Incorreta")
def save_file_csv_from_request(request_received,
field_request,
path_folder,
discard_files_existing=True):
logger.info("Inicio do método [salvar_arquivo_csv_from_request]")
file = request_received.files[field_request]
if field_request not in request_received.files:
raise Exception("O campo [" + field_request +
"] NÃO está presente na requisição.....")
if file.filename == '' or file.filename is None:
raise Exception("Nenhum arquivo foi enviado na requisição...")
if file and not is_extension_csv(file.filename):
raise Exception(
"Extensão do arquivo enviado é inválida. Extensões permitidas" +
str(allowed_extensions))
if discard_files_existing:
logger.info(
"Movendo arquivos anteriores para a pasta de descartados...")
files_in_path = os.listdir(path_folder)
files_in_path = list(
filter(lambda f: f.endswith('.csv'), files_in_path))
files_in_path = [path_folder + "/" + f for f in files_in_path]
for index, filename in enumerate(files_in_path):
move_file_datetime(path_aquivo=files_in_path[index],
formato_hora='%Y%m%d',
formato_data='%H%M%S',
path_destino=path_app_data_unsuccessful)
filename = secure_filename(file.filename)
filename_path = path_folder + "/" + filename
file.save(filename_path)
file_name = file.filename.rsplit('.', 1)[0]
file_type = file.filename.rsplit('.', 1)[1]
logger.info(" Nome Arquivo: [" + file_name + "]")
logger.info("Extensão Arquivo: [" + file_type + "]")
return filename_path
def print_header():
logger.info("*****************************************************")
logger.info("************** INICIANDO PROCESSAMENTO **************")
logger.info("*****************************************************")
logger.info('')
def print_footer():
logger.info('')
logger.info('')
logger.info("*****************************************************")
logger.info("************** TERMINO DO PROCESSAMENTO *************")
logger.info("*****************************************************")
logger.info('')
logger.info('')
def rename_columns_dataframe(strSearch, strNew, df):
logger.info('Inicio do metodo [__rename_columns_dp_csv]')
df.columns = df.columns.str.replace(strSearch, strNew)
df.columns = df.columns.str.lower()
logger.info("Termino do metodo [__rename_columns_dp_csv]")
return df
##################################################################
app = Flask(__name__)
@app.before_first_request
def startup():
max_mb_upload = configuration.configuration_app.max_mb_file_upload * 1024 * 1024
app.config['ERROR_404_HELP'] = False
app.config['UPLOAD_FOLDER'] = path_app_model_uploaded
app.config['RESTPLUS_VALIDATE'] = True
app.config['MAX_CONTENT_LENGTH'] = max_mb_upload
blueprint = Blueprint('api', __name__)
api.init_app(blueprint)
app.register_blueprint(blueprint)
api.add_namespace(ns_log)
api.add_namespace(ns_status)
api.add_namespace(ns_model)
api.add_namespace(ns_auth)
api.version = "0.1"
threads = list()
t = threading.Thread(target=memory_monitor, name="MONITOR_MEMORIA")
threads.append(t)
t.start()
if __name__ == '__main__':
app.run(debug=False, use_reloader=False)
logger.info('>>>>> Starting development server at http://{}/ <<<<<'.format(
app.config['SERVER_NAME']))
def __init__(self):
logger.info("Objeto Instânciado.")
def upload_model(self, request_received):
logger.info("Inicio do metodo [upload_modelo]")
save_file_model_from_request(
request_received=request_received,
field_request="modelo",
path_folder_upload=path_app_model_uploaded)