def create_train_validation(source_path,
train_path,
validation_path,
train_ratio=0.75):
"""Split source file into train and validation data
Args:
source_path: source file path
train_path: Path to write train data
validation_path: Path to write validatio data
train_ratio: Train data ratio
Returns:
None
"""
nb_lines = num_lines(source_path)
nb_train = int(nb_lines * train_ratio)
counter = 0
with gfile.GFile(source_path, "r") as f, gfile.GFile(
train_path, "w") as tf, gfile.GFile(validation_path, "w") as vf:
for line in f:
if counter < nb_train:
tf.write(line)
else:
vf.write(line)
counter = counter + 1
def create_train_validation(source_path,
train_path,
validation_path,
train_ratio=0.75):
"""
Args:
source_path: 元データが保存されているPath
train_path: (D)を保存するPath
validation_path: (E)を保存するPath
train_ratio: 学習用に保存するデータの割合
Returns:
None
"""
nb_lines = num_lines(source_path)
nb_train = int(nb_lines * train_ratio)
counter = 0
with gfile.GFile(source_path, "r") as f, \
gfile.GFile(train_path, "w") as tf, gfile.GFile(validation_path, "w") as vf:
for line in f:
if counter < nb_train:
tf.write(line)
else:
vf.write(line)
counter = counter + 1
def handle_content_message(event):
if isinstance(event.message, ImageMessage):
ext = 'jpg'
else:
return
message_content = line_bot_api.get_message_content(event.message.id)
with tempfile.NamedTemporaryFile(dir=static_tmp_path, prefix=ext + '-', delete=False) as tf:
for chunk in message_content.iter_content():
tf.write(chunk)
tempfile_path = tf.name
dist_path = tempfile_path + '.' + ext
dist_name = os.path.basename(dist_path)
os.rename(tempfile_path, dist_path)
predict_message = json.dumps(poster_predict(
os.path.join('static', 'tmp', dist_name)))
line_bot_api.reply_message(
event.reply_token, [
# TextSendMessage(text='Save content.'),
# TextSendMessage(text=request.host_url + \
# os.path.join('static', 'tmp', dist_name)),
TextSendMessage(text=predict_message)
])
def handle_content_message(event):
global graph
with graph.as_default():
message_content = line_bot_api.get_message_content(event.message.id)
with tempfile.NamedTemporaryFile(dir=static_tmp_path,
prefix="jpg" + '-',
delete=False) as tf:
for chunk in message_content.iter_content():
tf.write(chunk)
tempfile_path = tf.name
dist_path = tempfile_path + '.' + "jpg"
dist_name = os.path.basename(dist_path)
os.rename(tempfile_path, dist_path)
filepath = os.path.join('static', 'tmp',
dist_name) # 送信された画像のパスが格納されている
# 以下、送信された画像をモデルに入れる
img = image.load_img(filepath,
target_size=(32, 32)) # 送信された画像を読み込み、リサイズする
img = image.img_to_array(img) # 画像データをndarrayに変換する
data = np.array([img]) # model.predict()で扱えるデータの次元にそろえる
result = model.predict(data)
predicted = result.argmax() # 予測結果が格納されている
pred_answer = "これは" + class_label[predicted] + "です。"
line_bot_api.reply_message(event.reply_token,
TextSendMessage(text=pred_answer))
def get_weight(project_id, params=None):
res = requests.get(f'{base_url}/project/{project_id}/project/weight',
params=params,
headers={'AUTH': get_auth_header()})
with TemporaryFile() as tf:
tf.write(res.content)
_ = tf.seek(0)
weight = np.load(tf, allow_pickle=True)
return weight
def result_learning(project_id, params=None):
res = requests.get(f'{base_url}/project/{project_id}/result',
params=params,
headers={'AUTH': get_auth_header()})
with TemporaryFile() as tf:
tf.write(res.content)
_ = tf.seek(0)
weight = np.load(tf, allow_pickle=True)
# if res.status_code not in [200, 201, 204]:
# raise exc.ResponseException(res)
return weight
def leaveOneOut(self,
nb_epoch=250,
batch_size=128,
init_ler=0.05,
final_ler=0.005,
brain="tka002"):
dec = (final_ler / init_ler)**(1 / nb_epoch)
traindata = DataManager(self.getDataFunc)
for i in range(len(self.brains)):
test = [self.brains[i]]
train = self.brains[0:i] + self.brains[i + 1:len(self.brains)]
if (brain != test[0]):
continue
print(train)
print("Starting cv number", i, " - ", test[0], "out of",
len(self.brains))
traindata.setTrain(train)
cv_history = []
ler = init_ler
start_time = time.time()
for j in range(nb_epoch):
print("Starting epoch:", j + 1, "/", nb_epoch)
print("Genrating new training data")
d = traindata.getData()
sgd = SGD(lr=ler, decay=0, momentum=0.0, nesterov=False)
self.model.compile(loss='binary_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
tr_h = self.model.fit(d[0],
d[1],
batch_size=batch_size,
nb_epoch=1,
verbose=2)
print("train_loss", tr_h.history["loss"][0])
cv_history.append(tr_h.history["loss"][0])
ler *= dec
elapsed_time = time.time() - start_time
print("#################################################")
print("\tTime training (", i, "):", elapsed_time)
print("#################################################")
self.model.save("../models/model_" + self.model_name + "_for_" +
test[0] + ".mdl")
with open(
"../models/hist_" + self.model_name + "_for_" + test[0] +
".json", "w") as tf:
tf.write(json.dumps(cv_history))
def handle_file_message(event):
message_content = line_bot_api.get_message_content(event.message.id)
with tempfile.NamedTemporaryFile(dir=static_tmp_path, prefix='file-', delete=False) as tf:
for chunk in message_content.iter_content():
tf.write(chunk)
tempfile_path = tf.name
dist_path = tempfile_path + '-' + event.message.file_name
dist_name = os.path.basename(dist_path)
os.rename(tempfile_path, dist_path)
line_bot_api.reply_message(
event.reply_token, [
TextSendMessage(text='Save file.'),
TextSendMessage(text=request.host_url + os.path.join('static', 'tmp', dist_name))
])
def on_data(self, data):
try:
self.num_tweets += 1
if (self.num_tweets - 1 < self.wanted_tweets):
with open(self.fetched_tweets_filename, 'a') as tf:
json_load = json.loads(data)
text = {'text': json_load['text']}
print(text)
tf.write(json.dumps(text))
tf.write('\n')
return True
else:
return False
except BaseException as be:
print("Error on data: %s" % str(be))
return True
def run_synthesis(args, checkpoint_path, output_dir, sentences):
metadata_filename = os.path.join(args.input_dir, 'train.txt')
print(hparams_debug_string())
synth = Synthesizer()
synth.load(checkpoint_path, gta=args.GTA)
wav = load_wav(args.reference_audio)
reference_mel = melspectrogram(wav).transpose()
with open(metadata_filename, encoding='utf-8') as f:
metadata = [line.strip().split('|') for line in f]
frame_shift_ms = hparams.hop_size / hparams.sample_rate
hours = sum([int(x[4]) for x in metadata]) * frame_shift_ms / (3600)
print('Loaded metadata for {} examples ({:.2f} hours)'.format(len(metadata), hours))
if args.GTA==True:
synth_dir = os.path.join(output_dir, 'gta')
else:
synth_dir = os.path.join(output_dir, 'natural')
#Create output path if it doesn't exist
os.makedirs(synth_dir, exist_ok=True)
os.makedirs(os.path.join(synth_dir, 'wavs/'), exist_ok=True)
print('starting synthesis')
with open(os.path.join(synth_dir, 'map.txt'), 'w') as file:
#for i, meta in enumerate(tqdm(metadata)):
#text = meta[5]
for i, text in enumerate(tqdm(sentences)):
mel_output_filename = synth.synthesize(text=text, index=i+1, out_dir=synth_dir, log_dir=None, mel_filename=None, reference_mel=reference_mel)
mels = np.load(mel_output_filename)
wav = audio.inv_mel_spectrogram(mels.T)
audio.save_wav(wav, os.path.join(synth_dir, 'wavs/speech-wav-{:05d}-mel.wav'.format(i+1)))
with open(os.path.join(synth_dir, 'wavs/speech-wav-{:05d}.txt'.format(i+1)), 'w') as tf:
tf.write(text)
if hparams.predict_linear:
# save wav (linear -> wav)
wav = audio.inv_linear_spectrogram(linear.T)
audio.save_wav(wav, os.path.join(synth_dir, 'wavs/speech-wav-{:05d}-linear.wav'.format(i+1)))
#file.write('{}|{}|{}|{}\n'.format(text, mel_filename, mel_output_filename, wav_filename))
print('synthesized mel spectrograms at {}'.format(synth_dir))
def handle_message(event):
global model
if model is None:
model = load_model("./static/linebot_model.h5")
global graph
with graph.as_default():
message_content = line_bot_api.get_message_content(event.message.id)
with tempfile.NamedTemporaryFile(dir=static_tmp_path,
prefix="jpg" + '-',
delete=False) as tf:
for chunk in message_content.iter_content():
tf.write(chunk)
tempfile_path = tf.name #ファイルが保存された場所を示す
dist_path = tempfile_path + '.' + "jpg"
dist_name = os.path.basename(dist_path) #file名だけを抽出してくる
os.rename(tempfile_path, dist_path)
# ユーザから送信された画像のパスが格納されている
filepath = os.path.join('./static', 'tmp', dist_name)
class_label = [
"熊", "猫", "鹿", "犬", "象", "蛙", "キリン", "ゴリラ", "馬", "ライオン", "猿",
"ウサギ", "アザラシ"
]
img = image.load_img(filepath,
target_size=(150, 150)) # 送信された画像を読み込み、リサイズする
img = image.img_to_array(img) # 画像データをndarrayに変換する
#img = np.array(img)
#img = (img-img.mean()) / img.std()
# モデルは4次元データを受け取るのでnp.array()にimgをリストとして渡し、4次元のデータにする
data = np.array([img])
result = model.predict(data)
predicted = result.argmax()
pred_answer = "この写真は" + class_label[
predicted] + "に似ているね！また一つ動物博士に近づいたよ！"
line_bot_api.reply_message(event.reply_token,
TextSendMessage(text=pred_answer))
def promptuser(lines):
"""Display <lines> to user in their favourite editor.
Then return the lines he entered ommiting empty lines and
ones beginning with a #."""
tf = tempfile.NamedTemporaryFile(delete=False)
for line in lines:
tf.write(line + '\n')
tf.close()
editor = os.getenv('EDITOR')
if editor == None:
editor = 'vi'
subp = subprocess.Popen([editor, tf.name])
subp.wait()
f = open(tf.name)
res = f.readlines()
f.close()
os.remove(tf.name)
res = map(lambda x: x.strip(), res) #strip
res = filter(None, res) #remove empty lines
res = filter(lambda x: x[0] != '#', res) #remove comment lines
return res
y_train,
batch_size=batch_size,
nb_epoch=1,
verbose=2)
print("train_loss", tr_h.history["loss"][0])
cv_history.append(tr_h.history["loss"][0])
ler *= dec
if j % 50 == 0:
final_model.save("../models/model_" + model_name + "_it_" +
str(j) + "_cv_" + str(i) + ".mdl")
final_model.save("../models/model_" + model_name + "_cv_" + str(i) +
".mdl")
with open("hist_" + model_name + "_" + str(i) + ".json", "w") as tf:
tf.write(json.dumps(cv_history))
quit()
#model = load_model("../models/model_0.mdl")
tr.reset()
### test stuff
tt = imm.ImageManager() # load training data
tt.init(test_brain)
tt.createSlices(step=step + 1)
tt.balance(bal_test)
tt.split(1) # we will select the hole brain
X_test_x = tt.getData(img_types, "2dx", inp_dim_2d)[0]
y_test = X_test_x[1]
def start_learning(project_id, params=None):
callback.stop_learning_tok = False
res = requests.get(f'{base_url}/project/{project_id}/task/get',
params=params,
headers={'AUTH': get_auth_header()})
res_json = res.json()
print(res_json)
if (not (res_json['is_successful'])): return 'FAIL'
occupy_task(project_id, {'task_index': res_json['task_index']})
model_url = res_json['model_url']
data_url = res_json['data_url']
label_url = res_json['label_url']
start_time = time.time()
with TemporaryFile() as tf:
tf.write(requests.get(url=model_url).content)
_ = tf.seek(0)
model = keras.models.load_model(h5py.File(tf, mode='r'))
with TemporaryFile() as tf:
tf.write(requests.get(url=data_url).content)
_ = tf.seek(0)
train_data = np.asarray(np.load(tf,
allow_pickle=True)).astype(np.float32)
with TemporaryFile() as tf:
tf.write(requests.get(url=label_url).content)
_ = tf.seek(0)
train_label = np.asarray(np.load(tf,
allow_pickle=True)).astype(np.float32)
init_weight = get_weight(project_id)
model.set_weights(init_weight.tolist())
task_index = int(res_json['task_index'])
epoch = int(res_json['epoch'])
batch_size = int(res_json['batch_size'])
valid_rate = float(res_json['valid_rate'])
print(train_data.shape)
if (task_index == -1):
validate(project_id)
return 'STOP'
try:
model.fit(train_data,
train_label,
batch_size=batch_size,
epochs=epoch,
validation_split=valid_rate,
callbacks=[callback],
verbose=2)
except ValueError as e:
report_error(project_id=project_id, params={'error_message': e})
return 'ERROR'
except TypeError as e:
report_error(project_id=project_id, params={'error_message': e})
return 'ERROR'
if callback.stop_learning_tok:
return 'STOP'
spent_time = time.time() - start_time
with TemporaryFile() as tf:
np.save(
tf,
np.array(model.get_weights(), dtype=object) -
np.array(init_weight, dtype=object))
_ = tf.seek(0)
update_success = update_learning(project_id=project_id,
params={
'task_index':
res_json['task_index'],
'spent_time': spent_time
},
gradient={'gradient': tf})
if (not (update_success)): return 'FAIL'
return 'SUCCESS'
Tmat = TUM_vec_to_Tmat(R, t)
# print('Tmat:',Tmat)
if (file.split('.')[0] == '000000'):
# print(type(tx))
tx = float(tx)
ty = float(ty)
tz = float(tz)
qx = float(qx)
qy = float(qy)
qz = float(qz)
qw = float(qw)
# the first line
tf.write(lines0)
tf.write('\n')
# the second line
tf.write('%f %f %f %f %f %f %f %f\n' %
(time, tx, ty, tz, qx, qy, qz, qw))
this_pose = Tmat
else:
this_pose = np.dot(Tmat, this_pose)
# print('this_pose:',this_pose)
tx = this_pose[0, 3]
ty = this_pose[1, 3]
tz = this_pose[2, 3]
rot = this_pose[:3, :3]
qw, qx, qy, qz = rot2quat(rot)
tf.write('%f %f %f %f %f %f %f %f\n' %
