def file_upload():
if request.method == 'POST':
global input_data
target = os.path.join(UPLOAD_FOLDER, 'uploads')
check_dir(target)
print("welcome to upload")
file = request.files['file']
destination = "/".join([target, file.filename])
print(f"filename : {file.filename}")
file.save(destination)
preprocess(filename=file.filename)
input_data = load_pickle_data('./data/output/sample_output.pkl')
return jsonify({'result': 'success'})
def prepare_data(path, word2idx, num_threads=8, **opts):
with tf.device("/cpu:0"):
enqueue_data, dequeue_batch = get_input_queues(
path, word2idx, batch_size=opts["batch_size"], num_threads=num_threads)
# TODO: put this logic somewhere else
input_ph = tf.placeholder_with_default(dequeue_batch, (None, None))
source, target, sequence_length = preprocess(input_ph)
return enqueue_data, input_ph, source, target, sequence_length
def prepare_data(path, word2idx, num_threads=8, **opts):
with tf.device("/cpu:0"):
enqueue_data, dequeue_batch = get_input_queues(
path, word2idx, batch_size=opts["batch_size"], num_threads=num_threads)
# TODO: put this logic somewhere else
input_ph = tf.placeholder_with_default(dequeue_batch, (None, None))
source, target, sequence_length = preprocess(input_ph)
return enqueue_data, input_ph, source, target, sequence_length
def main(args):
torch.manual_seed(args.seed)
model = Model()
model.cuda()
train_loader, verification_loader = preprocess(args.data_dir)
train(model, train_loader, verification_loader, args.epochs,
args.train_batch_sz, args.verification_batch_sz, args.save_path)
def main(args):
# # Device configuration
device = torch.device(
'cuda:{}'.format(args.gpu) if torch.cuda.is_available() else 'cpu')
num_epochs = 80
num_classes = 8
learning_rate = 0.08
num_views = 3
num_layers = 4
data_path = args.dir
file_list = [
'./data/train_web_content.npy', './data/train_web_links.npy',
'./data/train_web_title.npy', './data/test_web_content.npy',
'./data/test_web_links.npy', './data/test_web_title.npy',
'./data/train_label.npy', './data/test_label.npy'
]
aaa = list(map(os.path.exists, file_list))
if sum(aaa) != len(aaa):
print(
'Raw data has not been pre-processed! Start pre-processing the raw data.'
)
data_loader.preprocess(data_path)
else:
print('Loading the existing data set...')
# train_dataset = data_loader.Load_datasets('train', num_classes)
train_dataset = data_loader.Load_datasets('train', 8)
train_loader = DataLoader(train_dataset,
batch_size=32,
shuffle=True,
num_workers=4)
input_dims = np.array(train_dataset.data[0]).shape
model = CNN_Text(input_dims, [64, 32, 32, 32], [1, 2, 3, 4], num_classes,
0.5, num_layers, num_views).to(device)
model = model.double()
model.device = device
model.learning_rate = learning_rate
model.epoch = 0
if args.model != None:
model.load_state_dict(torch.load(args.mpodel))
print('Successfully load pre-trained model!')
# train the model until the model is fully trained
train_model(model, train_loader, num_epochs)
print('Finish training process!')
evaluation(model)
def recognize(self, audio_path):
preprocessed_audio = preprocess(audio_path, self.sample_rate, self.window_size, self.window_stride)
if self.cpu:
from PuzzleLib.CPU.CPUArray import CPUArray
inputs = CPUArray.toDevice(np.array([preprocessed_audio]).astype(np.float32))
else:
from PuzzleLib.Backend import gpuarray
inputs = gpuarray.to_gpu(np.array([preprocessed_audio]).astype(np.float16))
output = self.w2l(inputs).get()
output = np.vstack(output).astype(np.float32)
result = self.decoder.decode(output)
if not self.cpu:
from PuzzleLib.Backend.gpuarray import memoryPool
memoryPool.freeHeld()
del inputs, output
return result
import pandas as pd
import pprint
from data_loader import load_pickle_data, preprocess
from utils import listing_daterange, find_string_idx, hlight_term, analyze_docs
from lda.lda_trainer import LDATrainer
from lda.lda_corpus import *
from lda.lda_topic_modeling import TopicModeler
from lda.pipeline import Pipeline
if __name__ == '__main__':
preprocess('sample_100_news.tsv')
input_raw_data = load_pickle_data('./data/output/sample_output.pkl')
#prepare dictionary, corpus
tokens_list = input_raw_data['tokens']
bigram = create_bigram_model(tokens_list, min_count=5, threshold=50)
tokens_list = convert_to_bigram_tokens(tokens_list, bigram)
#modeling
trainer = LDATrainer(tokens_list=tokens_list, tf_idf=False)
# trainer.train(trainer.corpus, trainer.dictionary, num_topics=6, passes=5, workers=4, iterations=10, chunksize=50,
# save=True)
# trainer.fit_optimal_topic_model(trainer.corpus, trainer.dictionary, tokens_list,
# passes=30, workers=4, iterations=20, chunksize=400,
# limit=10, start=2, step=2)
topic_model = TopicModeler()
print(topic_model.sorted_topic_keyword(topic_model.topic_info, topic=1, _lambda=0.5))
# topic_model.show_topic_modeling()
def main():
opt = TrainOptions().parse()
print('point0, option parser finished')
''' 1. load_data, only select feature images,
default: load the gradient of DEM
return: min, max among all the input images
'''
frame_data, mask_data, minn, maxx = load_feature_data(opt.frame_path, opt.mask_path, gradient=False,dim=opt.input_shape)
print('point1, finished load data')
print('point2, shape frame mask', frame_data.shape, mask_data.shape)
'''2. split train_val_test:
input_train/val/test
label_train/val/test '''
input_train, input_test, label_train, label_test = train_test_split(
frame_data, mask_data, test_size=0.15, shuffle=False)
input_train, input_val, label_train, label_val = train_test_split(
frame_data, mask_data, test_size=0.1, shuffle=False)
print('point3, shape frame mask', input_train.shape, label_train.shape)
n_train, n_test, n_val = len(input_train), len(input_test), len(input_val)
print('***** #train: #test: #val = %d : %d :%d ******'%(n_train, n_test, n_val))
Data_dict = {
'train':[input_train.astype('float32'),
label_train.astype('float32')],
'val':[input_val.astype('float32'),
label_val.astype('float32')],
'test':[input_test.astype('float32'),
label_test.astype('float32')]
}
'''3. preprocess_data
-----
normalize all the data
'''
preprocess(Data_dict, minn, maxx, opt.input_shape)
mkdir(opt.result_path)
if opt.isTrain:
# the actual model
mkdir(opt.model_path)
define_model(Data_dict, opt)
else:
# test/ prediction
print('===========test==========')
test_model(Data_dict, opt)
# visualize result
img = Data_dict['test'][0][:,:,:,0]
real = np.load(opt.result_path + '/gt_labels.npy')
pred = np.load(opt.result_path + '/pred_labels.npy')
predicted_data = np.zeros(pred.shape)
for i in range(pred.shape[0]):
for j in range(pred.shape[1]):
for k in range(pred.shape[2]):
if (pred[i,j,k]>=0.5):
predicted_data[i,j,k] =1
else:
predicted_data[i,j,k] =0
for i in range(100):
visualize(opt.result_path,img,real,pred,predicted_data,i)