def app_principal():
""" This is the main method of application.
It start this app and control it
"""
start = dp.DataProcess()
t_a, h_a, h_s = start.data_in()
start.define_disease_table_data(t_a, h_a, h_s)
def setDfMonthData(self, startDate=None, endDate=None):
dpMonthObj = dp.DataProcess(self.code, self.name, period=dp.MONTH)
dpMonthObj.readData()
self.dpMonthObj = dpMonthObj
if (os.path.exists(dpMonthObj.dataGenCsvFile)):
self.dfMonthGenData = pd.read_csv(dpMonthObj.dataGenCsvFile,
encoding=sd.UTF_8,
dtype={'code': str})
self.dfMonthFilterData = self.filterData(self.dfMonthGenData,
startDate, endDate)
else:
print(
'[Function:%s line:%s stock:%s] Error: File %s is not exist' %
(self.setDfWeekData.__name__, sys._getframe().f_lineno,
self.code, dpMonthObj.dataGenCsvFile))
sys.exit()
if (os.path.exists(dpMonthObj.signalReportFile)):
self.dfMonthSignalData = pd.read_csv(dpMonthObj.signalReportFile,
encoding=sd.UTF_8,
dtype={'code': str})
else:
print(
'[Function:%s line:%s stock:%s] Error: File %s is not exist' %
(self.setDfWeekData.__name__, sys._getframe().f_lineno,
self.code, dpMonthObj.signalReportFile))
sys.exit()
self.reportPath = dpMonthObj.dataPath + self.code + '_exchange_report.csv'
def handleCloudMongoData(self, uri, hostPort, file_data, idCollec):
client = pym.MongoClient(uri, hostPort, connectTimeoutMS=30000, socketTimeoutMS=None, socketKeepAlive=True)
db = client.get_database()
print" \n DB structure: ", db
print" \n DB name: ", db.name
print" \n Collection client: ", db.client
if idCollec == 1: # dadosNumSensores2
dadosNumSensores2 = db['dadosNumSensores2']
try:
dadosNumSensores2.insert(file_data)
except IOError:
print("\n\n Erro de insersao de dados na colecao dadosNumSensores2")
self.closeCMConection()
exit(0)
elif idCollec == 2: # dadosVerificSensores2
dadosVerificSensores2 = db['dadosVerificSensores2']
try:
dadosVerificSensores2.insert(file_data)
except IOError:
print("\n\n Erro de insersao de dados na colecao dadosVerificSensores2")
self.closeCMConection()
exit(0)
elif idCollec == 3: # controle2
controle2 = db['controle2']
try:
controle2.insert(file_data)
except IOError:
print("\n\n Erro de insersao de dados na colecao controle2")
self.closeCMConection()
exit(0)
else: # recuperar o tempo de resposta: idCollec = 4
# docum = db.get_collection('controle').find({"tempo":{$gte:5}})
try:
docum = db.get_collection('controle').find().pretty()
d = dict(docum)
tempoUser = d.get('tempo') # recupera o valor do campo tempo
dpo = Dp.DataProcess()
dpo.tempoAtual = dpo.converterTempo(tempoUser)
except IOError:
print("\n\n Erro na selecao de documentos da colecao controle")
self.closeCMConection()
exit(0)
def main():
# Input size of each steps
input_size = args.num_joint * args.coord_dim
# Loading data
DataLoader = DataProcess.DataProcess(
path=args.data_path,
batch_size=args.batch_size,
num_joint=args.num_joint,
coord_dim=args.coord_dim,
# input_size=input_size,
decoder_steps=args.decoder_steps,
model=args.model) # TODO
# Build graph & Train/Test
solver = Solver()
if args.test:
solver.test(args=args, DataLoader=DataLoader)
else:
# Biuld net
if args.model == "cnn":
net = Model(
name="resnet",
layer_n=3,
in_shape=[args.in_frames, args.num_joint, args.coord_dim],
out_shape=[args.out_band],
num_steps=args.epochs * DataLoader.Get_num_batch(
DataLoader.train_set['source'], args.in_frames),
lr=args.learning_rate)
elif args.model == "rnn":
# Build Sequence to Sequence Model
net = Model(seq_length=args.in_frames,
out_length=args.out_band,
rnn_size=args.rnn_size,
num_layers=args.num_layers,
batch_size=args.batch_size,
input_size=input_size,
decoder_steps=args.decoder_steps,
num_steps=args.epochs * DataLoader.Get_num_batch(
DataLoader.train_set['source'], args.in_frames),
lr=args.learning_rate)
solver.train(args=args, DataLoader=DataLoader, net=net)
solver.test(args=args, DataLoader=DataLoader)
def __init__(self):
# Instância a classe que irá ler os arquivos de entrada do programa como stop-word e tags
inputFiles = ReadInputFiles.ReadInputFiles()
# Instância da classe generateID que gera os id para os registros a serem adicionados no banco
generateID = GenerateID.GenerateID()
# Chama o método que cria todas os dicionários com os valores.
inputFiles.mountDicts()
writeLogFile = WriteLogFile.WriteLogFile()
#Cria uma instância da classe Bolsa DAO, a qual fica realizando as consultas no banco
bolsa = BolsaDAO.BolsaDAO(writeLogFile)
pool_sema = threading.BoundedSemaphore(value=1)
self.generateID = GenerateID.GenerateID()
dicionario = Dict.Dict(bolsa, pool_sema, writeLogFile)
numberOfThread = 4
#Pega a quantidade de linhas do banco de dados
rowCount = bolsa.getTableRowCount()
areaDeConhecimento = self.montaHashAreaCohecimento(bolsa)
# A quantidade de registros que serão trazidos do banco de dados por vez.
offset = 5000
# realiza a quantidade de vez para que se rode o algoritmo no banco todo.
cycles = math.ceil(rowCount / offset)
i = inputFiles.readOnFileCycle()
inputFiles.readOnFileHashWord(dicionario.wordDict)
inputFiles.readOnFileHashStem(dicionario.stemDict)
for i in range(0, cycles):
#pegando as 1000 registros do banco de cada vez
rows = bolsa.getBolsaOffsetLimit(offset, offset * i)
threads = []
k = 0
while k < len(rows):
for l in range(0, numberOfThread):
if k < len(rows):
thread = DataProcess.DataProcess(
inputFiles.stopWordBannedList,
inputFiles.stopWordConnectList,
inputFiles.tagTrigram, inputFiles.tagBigram, bolsa,
rows[k], pool_sema, self.generateID,
areaDeConhecimento, dicionario)
threads.append(thread)
thread.start()
k = k + 1
for l in range(0, len(threads)):
threads[l].join()
del threads[:]
print("..::Já foram processadas ->" + str(offset * (i + 1)))
bolsa.mysql.commit()
if i > 0 and i % 10 == 0:
dicionario.saveStemAndWordDict(i)
#fechando os arquivos de log que contem os erros de insercao de cada tabela
bolsa.fileSource.close()
bolsa.fileWord.close()
bolsa.fileStem.close()
dicionario.saveStemAndWordDict(i)
import glob
import os
import shutil
import numpy as np
import subprocess
import easygui
import tkinter as tk
from tkinter import messagebox
#local
import DataProcess as dp
dp = dp.DataProcess()
def main():
dirPath = r"C:\Users\紅林亮平\Desktop\【Input to excel】"
fileCreater(dirPath)
def openExplorer(path):
FILEBROWSER_PATH = os.path.join(os.getenv('WINDIR'), 'Explorer.exe')
subprocess.run([FILEBROWSER_PATH, '/select,', os.path.normpath(path)])
def getDirPath():
return easygui.diropenbox(title="変換するフォルダを選択")
# def forDemoRmOutputDir():
[-8*h**3, -h**3, 0, h**3, 8*h**3],
[16*h**4, h**4, 0, h**4,16*h**4]])
if args.degree == 1:
# First Derivative
B = np.array([0, 1, 0, 0, 0])
else:
# Second Derivative
B = np.array([0, 0, 2, 0, 0])
X = np.linalg.solve(A, B)
# Loading data
DataLoader = DataProcess.DataProcess(path=args.data_path,
batch_size=args.batch_size,
num_joint=args.num_joint,
coord_dim=args.coord_dim,
decoder_steps=args.decoder_steps) # TODO
print(DataLoader.valid_set['source']['squat_front'].shape)
data = DataLoader.valid_set['source']['squat_front'][:,0,1]
result = np.zeros((len(data)-4,), np.float32)
print(data.shape, result.shape)
for i in range(len(result)):
mini_data = data[i:i+5]
# Curve fitting
# A.T * A * X = A.T * B
# CA = np.array([[1, -2, (-2)**2, (-2)**3],
# [1, -1, (-1)**2, (-1)**3],
# [1, 0, ( 0)**2, ( 0)**3],
from DataProcess import *
from XGBoostModel import *
from FeatureEngineering import *
from TimeCost import *
if __name__ == '__main__':
mode = Const.VALID_MODE
mode = Const.PREDICT_MODE
tc = TimeCost()
dp = DataProcess(mode)
df, dft = dp.data_input(Const.TRAIN_FILE_NAME, Const.TEST_FILE_NAME)
tc.print_event()
fe = FeatureEngineering(mode)
df, dft = fe.feature_process(df, dft)
tc.print_event()
X_train, y_train, X_valid, y_valid, X_test = dp.get_split_data(df, dft)
tc.print_event()
xgb = XGBoostModel(mode)
result = xgb.train_model(X_train, y_train, X_valid, y_valid, X_test)
tc.print_event()
if not mode:
dp.transform_index(result)
# xgb.load_model()
s=5,
color=(0, 1, 0),
label='Recall')
plt.scatter(EPOCHS,
F_1_t_n,
marker='o',
s=5,
color=(0, 0, 1),
label='F1-Measure')
plt.legend()
plt.show()
if __name__ == '__main__':
# 分别处理训练集、验证集、测试集文件
DataProcess.DataProcess(train, train_out)
DataProcess.DataProcess(validation, validation_out)
DataProcess.DataProcess(test, test_out)
# 将原始词向量文件精简
VecPre.VecPre(wordvector, wordvector_out)
# 读取训练集、验证集、测试集
train_set = open(train_out, encoding='utf-8').readlines()
validation_set = open(validation_out, encoding='utf-8').readlines()
test_set = open(test_out, encoding='utf-8').readlines()
# 读取词向量
word_vector = open(wordvector_out, encoding='utf-8').readlines()
# 建立词向量的索引
vector_to_idx = {}
for i in range(len(word_vector)):
vector_to_idx[word_vector[i].split()[0]] = len(vector_to_idx)
# 建立词的索引
f.write('\n'.join(sys.argv[1:]))
# save information
now = datetime.datetime.now()
current_time = '{:04d}_{:02d}_{:02d}_{:02d}{:02d}{:02d}\n'.format(
now.year, now.month, now.day, now.hour, now.minute, now.second)
with open('./model/info.txt', 'w') as out_file:
out_file.write(current_time)
out_file.write(args.info)
if __name__ == '__main__':
# Input size of each steps
input_size = args.num_joint * args.coord_dim
# Loading data
DataLoader = DataProcess.DataProcess(path=args.data_path,
batch_size=args.batch_size,
input_size=input_size,
decoder_steps=args.decoder_steps)
if not args.test:
train_batch_generator = DataLoader.Batch_Generator(
DataLoader.train_set['source'], DataLoader.train_set['target'],
args.in_frames, args.out_band)
(_, _, num_batch) = next(train_batch_generator)
# Define graph
train_graph = tf.Graph()
with train_graph.as_default():
# Get placeholder
(input_data, targets, keep_rate) = SegmentModel.get_inputs()
# Build Sequence to Sequence Model
