def saveCurrentPollutionData():
response = aqicn.getAllPollutionData()
save_path = config['AQICN']['SAVE_PATH']
utils.saveData(save_path,
"aqicn_index_" + utils.getCurrentDayTime() + ".json",
json.dumps(response, sort_keys=False, indent=2))
return jsonify(response)
def saveToLib(crv=None, shapeName=None):
'''Saves the shape data to a shape file in the SHAPE_LIBRARY_PATH directory'''
crvShape = getShape(crv=crv)
path = os.path.join(SHAPE_LIBRARY_PATH,
re.sub("\s", "", shapeName) + ".json")
for shapeDict in crvShape:
shapeDict.pop("colour", None)
utils.saveData(path, crvShape)
def exit(self): try: for x in datapipe.channelDict: for user in x.users: user = datapipe.getUser(user).name datapipe.lastseenDict[user]=datetime.datetime.now() except IndexError: pass utils.saveData(self.lastseenDict, 'lastseen')
def main():
#load the JPEG images or the npz file
#data, labels = load_images() #un-comment if want to load JPEG images into numpy
#saveData('data/data.npz', data, labels) #un-comment if want to save loaded JPEG images as npz for faster loading later
data, labels = loadData(
'data/data.npz'
) #load images from npz, much faster then loading from JPEGS each time
#split dataset into train, val, and test ... split into 70/30 and then 70/15/15
print('Splitting data into 70/15/15 train, val, and test sets.')
train, testval, train_labels, testval_labels = train_test_split(
data,
labels,
test_size=0.30,
random_state=42,
shuffle=True,
stratify=labels)
test, val, test_labels, val_labels = train_test_split(
testval,
testval_labels,
test_size=0.50,
random_state=42,
shuffle=True,
stratify=testval_labels)
del data, labels, testval, testval_labels #free up memory
#check imbalance
print('Train Shape:', train.shape, 'Train Labels Shape:',
train_labels.shape)
print('Validation Shape:', val.shape, 'Validation Labels Shape:',
val_labels.shape)
print('Test Shape:', test.shape, 'Test Labels Shape:', test_labels.shape)
plotClassDist(train_labels, 'Train Class Distribution')
plotClassDist(val_labels, 'Validation Class Distribution')
plotClassDist(test_labels, 'Test Class Distribution')
#random oversample train set and recheck balance
ovs_data, ovs_labels = randomOversample(train, train_labels)
print('OVS Data Shape:', ovs_data.shape, 'OVS Labels Shape:',
ovs_labels.shape)
plotClassDist(ovs_labels, 'Train Class Distribution (ROS)')
#oversample using smote on train set and recheck balance
smt_data, smt_labels = smoteOversample(train, train_labels)
print('SMT Data Shape:', smt_data.shape, 'SMT Labels Shape:',
smt_labels.shape)
plotClassDist(smt_labels, 'Train Class Distribution (SMOTE)')
#save untouched and oversampled data as npz along with val and test
saveData('data/train.npz', train, train_labels)
saveData('data/ros_data.npz', ovs_data, ovs_labels)
saveData('data/smt_data.npz', smt_data, smt_labels)
saveData('data/val.npz', val, val_labels)
saveData('data/test.npz', test, test_labels)
def main():
path2data = "data/post-process/data.pkl"
path2wordFreqDict = "data/post-process/wordfreq.pkl"
path2pretrainWordEmbedding = "data/word-embedding/glove.vtb.100d.txt"
devRatio = 0.1
path2dirOut = "data/nlm-input/"
data = utils.loadPKLFile(path2data)
wordFreqDict = utils.loadPKLFile(path2wordFreqDict)
vocab, invertVocab = buildVocabIndex(wordFreqDict)
print "Vocabulary Size:", len(vocab) + 1
input, inputLen, output = makeNeuralLMDataFormat(data, vocab)
input = convertList2NumpyArray(input)
inputLen = convertList2NumpyArray(inputLen)
output = convertList2NumpyArray(output)
# Split
input, inputLen, output = utils.unisonShuffleNumpyArrays(
input, inputLen, output)
totalIns = input.shape[0]
nbDev = int(totalIns * devRatio)
train = (input[:-nbDev], inputLen[:-nbDev], output[:-nbDev])
dev = (input[-nbDev:], inputLen[-nbDev:], output[-nbDev:])
# Word Embedding
pretrainWordEmbedding = utils.loadPretrainWordEmbedding(
path2pretrainWordEmbedding)
wordEmbeddingMatrix = makeWordEmbeddingMatrix(pretrainWordEmbedding, vocab,
100)
utils.saveData(path2dirOut, 'train', wordEmbeddingMatrix, *train)
utils.saveData(path2dirOut, 'dev', None, *dev)
utils.savePKLFile(os.path.join(path2dirOut, 'vocab.pkl'), vocab)
utils.savePKLFile(os.path.join(path2dirOut, 'invert-vocab.pkl'),
invertVocab)
print 'Finished!'
def oninfo(ev,server,plugin):
if ev["sender"] != False and ev["content"].startswith('!!loc'):
if re.match(r"^!!loc help$", ev["content"]):
printhelp(server)
elif re.match(r"^!!loc save$", ev["content"]):
global locs
utils.saveData('location', locs)
server.tell(ev["sender"], CC('[LOC] ','b'), CC('保存完毕', 'e'))
elif re.match(r"^!!loc add \S+ -?\d+ -?\d+ -?\d+ (-1|0|1)$", ev["content"]):
add(server, ev)
elif re.match(r"^!!loc add \S+ here$", ev["content"]):
addHere(server, ev)
elif re.match(r"^!!loc del \S+$", ev["content"]):
delete(server, ev)
elif re.match(r"^!!loc conv \S+$", ev["content"]):
conv(server, ev)
elif re.match(r"^!!loc ex$", ev["content"]):
getAllVoxel(server, ev)
elif re.match(r"^!!loc \S+$", ev["content"]):
get(server, ev)
elif re.match(r"^!!loc$", ev["content"]):
getAll(server, ev)
else:
server.tell(ev["sender"], CC('[LOC] ','b'), CC('输入无效，使用 !!loc help 查看帮助', 'c'))
def onunload(ev,sender,plugin):
if ev["name"] == name:
global locs
utils.saveData('location', locs)
elif 8 == sampRateMode:
imfFileExt = imfFileExt + '08'
else:
print('Invalid Sampling Rate Mode:', sampRateMode)
sys.exit()
savefilename = dataSaveFolder + '/' + filename.replace(
'.wav', imfFileExt)
# create folder if not existed
if not os.path.exists(os.path.dirname(savefilename)):
os.makedirs(os.path.dirname(savefilename))
inputShape = np.shape(imfFeatures)
# print(inputShape)
if inputShape[1:] == (4, 6, numberOfImfs):
# save imf feature input file
utils.saveData(imfFeatures, savefilename)
print('Prepared: ' + savefilename, inputShape)
if 48 == sampRateMode:
# save spectogram input file
inputShape = np.shape(spectos)
# print(inputShape)
savefilename = dataSaveFolder + '/' + filename.replace(
'.wav', '.spct48')
utils.saveData(spectos, savefilename)
print('Prepared: ' + savefilename, inputShape)
elif 8 == sampRateMode:
pass
else:
print('Invalid Sampling Rate Mode:', sampRateMode)
sys.exit()
else:
def onunload(ev, server, plugin):
if ev["name"] == name:
utils.saveData('gamemode', gm)
def saveResult(self):
utils.saveData(self.RESULT, self.OUTPUT_FILE)
self.LOG("Results saved to: " + self.OUTPUT_FILE)
def test(args):
my_model = RCAN(args) # model.RDN()
save = saveData(args)
dataloader = get_dataset(args)
my_model.cuda()
# my_model.eval()
model_path = os.path.join(args.model_savepath, args.model_name)
# my_model.load_state_dict(torch.load(model_path))
my_model = save.load_model(my_model, model_path)
for i, (lr_in, name) in enumerate(dataloader):
# _,_,w,h =lr_in.shape
# out_img =torch.ze 切图
# lr_in_ = lr_in.numpy()
_, _, w, h = lr_in.shape
# print(_,_,w,h)
# out_img = np.zeros((1, 3, w, h))
# in_img1 = np.zeros((1, 3, int(w / 3), int(h / 2)))
# in_img2 = np.zeros((1, 3, int(w / 2), int(h / 2)))
# for i in range(5):
# img_hr_out=np.zeros((3,w*4,h*4))
# for j in range(10):
# img_hr_out=np.zeros((3,w*4,h*4))
# z=0
# in_img1 = np.zeros((6,3, 90,60))
# for i_w in range(3):
# for i_h in range(2):
# in_img1[z,:,:,:] =lr_in[0,:,(i_w)*(int(w /3)):(i_w+1)*(int(w /3)),(i_h)*(int(h /8)):(i_h+1)*(int(h /8))]
# z=z+1
# in_img1
# in_img2 = lr_in[:, :, 0:w, int(h / 2):]
# in_img1=torch.from_numpy(in_img1)
# in_img2 = torch.from_numpy(in_img2)
in_img1 = lr_in.cuda().float() #, volatile=False)
in_img1 = my_model(in_img1)
in_img1 = in_img1[0]
img_hr_out = in_img1.cpu().data.numpy()
# z=0
# for i_w in range(3):
# for i_h in range(2):
# img_hr_out[:,(i_w)*(int(w /3))*4:(i_w+1)*(int(w /3))*4,4*(i_h)*(int(h /8)):4*(i_h+1)*(int(h /8))] = in_img1[z,:,:,:]
# z=z+1
img_hr_out = img_hr_out.transpose((1, 2, 0))
img_hr_out = img_hr_out
img_hr_out = np.ceil(img_hr_out * 256)
img_hr_out[img_hr_out > 255] = 255
img_hr_out[img_hr_out < 0] = 0
# img_hr_out1 = np.zeros((1080,1920,3))
out_i = i // 100
i_i = i % 100
# img_hr_out1 =cv2.imread(args.result_SR_Dir + "/sr/"+str(out_i) + '/%05d_sr.bmp' % (i_i))
#拼图片
# img_hr_out1[360:2*360,0:960,:]=img_hr_out
if not os.path.exists(args.result_SR_Dir + "/sr/" + str(out_i)):
os.mkdir(args.result_SR_Dir + "/sr/" + str(out_i))
cv2.imwrite(
args.result_SR_Dir + "/sr/" + str(out_i) + '/%05d_sr.bmp' % (i_i),
img_hr_out)
foldername = os.path.basename(folderpath).split('_')
foldername[0] = 'inputs'
dataSaveFolder = os.path.dirname(folderpath)+'/'+'_'.join(foldername)
inputFile = []
inputFile2 = []
swappedInstfArr = np.array(instfArr).swapaxes(0, 1)
swappedMagArr = np.array(mags).swapaxes(0, 1)[:-1]
print(swappedInstfArr.shape, swappedMagArr.shape)
for i in range(len(swappedInstfArr)):
inputFile.append({'instantFrequencies': swappedInstfArr[i], 'magnitudes': swappedMagArr[i]})
inputFile2.append([swappedInstfArr[i], swappedMagArr[i]])
savefilename = dataSaveFolder+'/'+filename+'.inp'
if not os.path.exists(os.path.dirname(savefilename)):
os.makedirs(os.path.dirname(savefilename))
print(np.shape(inputFile))
utils.saveData(inputFile, savefilename)
utils.saveData(np.array(inputFile2), savefilename+'2')
print(filename+'.inp prepared.')
print('----------------------')
print('plotting...')
if plotSpec:
specsAspectRatio = specs.shape[0] / specs.shape[1] # for better print size and look
# Make plot with vertical (default) colorbar
fig, ax = plt.subplots(figsize=(15 * specsAspectRatio, 15))
data = np.swapaxes(specs, 0, 1)
data = np.ma.masked_values(data, 0)
cmap = plt.cm.magma
def saveCache(self):
utils.saveData(self.RESULT, self.CACHE_FILE)
t0 = time.time()
for i in range(epochs):
print("Epoch", i)
for img, lbl, idx in zip(imgs, binLbls, range(NTRAIN)):
train(img, lbl)
t1 = time.time()
predictedClasses = predictMatrix(imgsTest)
nErrorTest = NTEST - np.sum(predictedClasses == lblsTest)
print("Test errors:", nErrorTest, "%:", nErrorTest / NTEST * 100.0)
print("Training time:", (t1 - t0))
np.savez("weights/" + "backSoft.npz",
W1=W1.get_value(),
b1=b1.get_value(),
W2=W2.get_value(),
b2=b2.get_value())
nErrorTrain = NTRAIN - np.sum(predictMatrix(imgs) == lbls)
print("Train errors:", nErrorTrain, "%:", nErrorTrain / NTRAIN * 100)
algDescription = "Algoritmo con backpropagation en una NN con una capa oculta de \n"
algDescription += str(
nHidden
) + " neuronas con función de activación la tanh de Lecun. Y una capa de\n"
algDescription += "salida tipo softmax. Usando una tasa de aprenzidaje de" + str(
lr) + "\n y dando "
algDescription += str(epochs) + " al conjunto de train."
saveData(algDescription, predictMatrix(imgs), predictMatrix(imgsTest), t1 - t0,
"b1withLeCunActFun.txt")
def onunload(ev, server, plugin):
if ev["name"] == name:
utils.saveData("backup", data)
def getdata(self, code):
data = utils.doParse(code)
utils.saveData(db, data)
return simplejson.dumps(data, cls=utils.MyJSONEncoder)
def magic(model_name,
opt,
lrs,
fc_layers=(1024, 1024),
batch_size=64,
dropout=0.5,
MAX_EPOCHS=100,
dry_run=False,
lr=0.0001,
acf='relu',
pred_acf='softmax',
model_from_file=False,
model_json="",
model_w_file=""):
#K.variable(lr, name='lr')
opt_name = opt[0]
opt = opt[1]()
lrs_name = lrs[0]
#model_name_save = f"{model_name}_detect_FULL__OPT_{opt_name}___LRS_{lrs_name}_final_1024"
model_name_save = f"{model_name}_detect_FULL__OPT_{opt_name}___LRS_{lrs_name}_final_4096"
print("Training for " + model_name_save)
if dry_run:
return
""" big data set
TRAIN_DIR = "data/Train/flower_splits/train"
VAL_DIR = "data/Train/flower_splits/val"
TEST_DIR = "data/Train/flower_splits/test"
"""
TRAIN_DIR = "data/Train/detect_train"
VAL_DIR = "data/Validation/detect_train"
#TEST_DIR = "data/TrainBig/flower/test"
img_width = 224
img_height = 224
img_channels = 3
#datagen = ImageDataGenerator(rescale=1./255)
train_datagen = ImageDataGenerator(
rescale=1. / 256,
horizontal_flip=True,
vertical_flip=True,
# shear_range=,
zoom_range=0.05,
width_shift_range=0.05,
height_shift_range=0.05,
rotation_range=15)
val_datagen = ImageDataGenerator(rescale=1. / 256)
#datagen = Augmentator._get_ImageDataGenerator()
train_gen = train_datagen.flow_from_directory(directory=TRAIN_DIR,
target_size=(img_width,
img_height),
color_mode='rgb',
batch_size=batch_size,
class_mode='categorical',
shuffle=True,
seed=seed)
val_gen = val_datagen.flow_from_directory(directory=VAL_DIR,
target_size=(img_width,
img_height),
color_mode='rgb',
batch_size=batch_size,
class_mode='categorical',
shuffle=True,
seed=seed)
# train_gen.n gets number of samples. // => floor division
TRAIN_STEP_SIZE = train_gen.n // batch_size
VAL_STEP_SIZE = val_gen.n // batch_size
# set up model
# dataset as input - used to extract number of classes and inputsize
if model_from_file:
model = get_model_from(json_file_name=model_json,
h5_file_name=model_w_file,
acf=acf,
pred_acf=pred_acf)
else:
model = get_model(model_name,
dataset=train_gen,
dropout=dropout,
fc_layers=fc_layers,
acf=acf,
pred_acf=pred_acf)
model_json = model.to_json()
with open(model_name_save + ".json", "w") as json_file:
json_file.write(model_json)
model.compile(optimizer=opt,
loss="categorical_crossentropy",
metrics=['accuracy'])
#K.set_value(model.optimizer.lr, lr)
lrs_obj = lrs[1]()
callbacks_list = get_callbacks(model_name_save)
callbacks_list.append(lrs_obj)
fit_history = model.fit_generator(train_gen,
steps_per_epoch=TRAIN_STEP_SIZE,
validation_data=val_gen,
validation_steps=VAL_STEP_SIZE,
epochs=MAX_EPOCHS,
callbacks=callbacks_list,
verbose=verbose)
print('Saving final weights...')
model.save_weights("dl_weights/" + model_name_save + timestamp +
"final.h5",
overwrite=True)
# ----------- CREATING PLOTS AND SAVING DATA -----------
DIR_TO_SAVE_IN = "results/plots/" + model_name + '/' + timestamp + '/'
if not os.path.exists(DIR_TO_SAVE_IN):
os.makedirs(DIR_TO_SAVE_IN)
utils.createPlot(fit_history.history['acc'],
modelName=model_name_save,
unit="Accuracy",
title="Model accuracy for " + model_name,
ylabel="Accuracy",
xlabel="Epochs",
folder=model_name + '/' + timestamp)
utils.createPlot(fit_history.history['loss'],
modelName=model_name_save,
unit="Loss",
title="Model loss for " + model_name,
ylabel="Loss",
xlabel="Epochs",
folder=model_name + '/' + timestamp)
utils.createPlot(fit_history.history['lr'],
modelName=model_name_save,
unit="LearnRate",
title="Model Learning Rate for " + model_name,
ylabel="Learning Rate",
xlabel="Epochs",
folder=model_name + '/' + timestamp)
utils.saveData(fit_history.history['acc'],
filepath=DIR_TO_SAVE_IN + "Accuracy.P")
utils.saveData(fit_history.history['loss'],
filepath=DIR_TO_SAVE_IN + "Loss.P")
utils.saveData(fit_history.history['lr'],
filepath=DIR_TO_SAVE_IN + "LearnRate.P")
lrs_obj = None
opt = None
model = None
val_gen = None
train_gen = None
val_datagen = None
train_datagen = None
print('Done...')
def processResult(result):
format = result.headers['Content-Type']
deviceBody = utils.deserialize(result, format)
utils.saveData(deviceBody, format)
t1 = time.time()
predictedClasses = predictMatrix(imgsTest)
nErrorTest = NTEST - np.sum(predictedClasses == lblsTest)
print("Test errors:", nErrorTest, "%:", nErrorTest / NTEST * 100.0)
print("Training time:", (t1 - t0))
fileName = "backSoftwithMomentumsLR" + str(lr) + "EPOCH" + str(
epochs) + "NHID" + str(nHidden) + "ETA" + str(eta) + ".npz"
np.savez("weights/" + fileName,
W1=W1.get_value(),
b1=b1.get_value(),
W2=W2.get_value(),
b2=b2.get_value())
nErrorTrain = NTRAIN - np.sum(predictMatrix(imgs) == lbls)
print("Train errors:", nErrorTrain, "%:", nErrorTrain / NTRAIN * 100)
algDescription = "Algoritmo con backpropagation en una NN con una capa oculta de \n"
algDescription += str(
nHidden) + " neuronas con función de activación sigmoidal. Y una capa de\n"
algDescription += "salida tipo softmax. Usando una tasa de aprenzidaje de" + str(
lr) + "\n y dando "
algDescription += str(
epochs
) + " al conjunto de train, usando además momentos con una tasa de" + str(eta)
saveData(
algDescription, predictMatrix(imgs), predictMatrix(imgsTest), t1 - t0,
"b1withMomentumsLR" + str(lr) + "EPOCH" + str(epochs) + "NHID" +
str(nHidden) + "ETA" + str(eta) + ".txt")
def unloading(ev, server, plugin):
global cfg
if ev["name"] == name:
utils.saveData("uuid", cfg)
def onunload(ev, server, plugin):
if ev["name"] == name:
utils.saveData("bot", {"bot": botlist, "loc": loclist, "log": botlog})