def create_train_validation(source_path, train_path, validation_path, train_ratio=0.75): """Split source file into train and validation data Args: source_path: source file path train_path: Path to write train data validation_path: Path to write validatio data train_ratio: Train data ratio Returns: None """ nb_lines = num_lines(source_path) nb_train = int(nb_lines * train_ratio) counter = 0 with gfile.GFile(source_path, "r") as f, gfile.GFile( train_path, "w") as tf, gfile.GFile(validation_path, "w") as vf: for line in f: if counter < nb_train: tf.write(line) else: vf.write(line) counter = counter + 1
def create_train_validation(source_path, train_path, validation_path, train_ratio=0.75): """ Args: source_path: 元データが保存されているPath train_path: (D)を保存するPath validation_path: (E)を保存するPath train_ratio: 学習用に保存するデータの割合 Returns: None """ nb_lines = num_lines(source_path) nb_train = int(nb_lines * train_ratio) counter = 0 with gfile.GFile(source_path, "r") as f, \ gfile.GFile(train_path, "w") as tf, gfile.GFile(validation_path, "w") as vf: for line in f: if counter < nb_train: tf.write(line) else: vf.write(line) counter = counter + 1
def handle_content_message(event): if isinstance(event.message, ImageMessage): ext = 'jpg' else: return message_content = line_bot_api.get_message_content(event.message.id) with tempfile.NamedTemporaryFile(dir=static_tmp_path, prefix=ext + '-', delete=False) as tf: for chunk in message_content.iter_content(): tf.write(chunk) tempfile_path = tf.name dist_path = tempfile_path + '.' + ext dist_name = os.path.basename(dist_path) os.rename(tempfile_path, dist_path) predict_message = json.dumps(poster_predict( os.path.join('static', 'tmp', dist_name))) line_bot_api.reply_message( event.reply_token, [ # TextSendMessage(text='Save content.'), # TextSendMessage(text=request.host_url + \ # os.path.join('static', 'tmp', dist_name)), TextSendMessage(text=predict_message) ])
def handle_content_message(event): global graph with graph.as_default(): message_content = line_bot_api.get_message_content(event.message.id) with tempfile.NamedTemporaryFile(dir=static_tmp_path, prefix="jpg" + '-', delete=False) as tf: for chunk in message_content.iter_content(): tf.write(chunk) tempfile_path = tf.name dist_path = tempfile_path + '.' + "jpg" dist_name = os.path.basename(dist_path) os.rename(tempfile_path, dist_path) filepath = os.path.join('static', 'tmp', dist_name) # 送信された画像のパスが格納されている # 以下、送信された画像をモデルに入れる img = image.load_img(filepath, target_size=(32, 32)) # 送信された画像を読み込み、リサイズする img = image.img_to_array(img) # 画像データをndarrayに変換する data = np.array([img]) # model.predict()で扱えるデータの次元にそろえる result = model.predict(data) predicted = result.argmax() # 予測結果が格納されている pred_answer = "これは" + class_label[predicted] + "です。" line_bot_api.reply_message(event.reply_token, TextSendMessage(text=pred_answer))
def get_weight(project_id, params=None): res = requests.get(f'{base_url}/project/{project_id}/project/weight', params=params, headers={'AUTH': get_auth_header()}) with TemporaryFile() as tf: tf.write(res.content) _ = tf.seek(0) weight = np.load(tf, allow_pickle=True) return weight
def result_learning(project_id, params=None): res = requests.get(f'{base_url}/project/{project_id}/result', params=params, headers={'AUTH': get_auth_header()}) with TemporaryFile() as tf: tf.write(res.content) _ = tf.seek(0) weight = np.load(tf, allow_pickle=True) # if res.status_code not in [200, 201, 204]: # raise exc.ResponseException(res) return weight
def leaveOneOut(self, nb_epoch=250, batch_size=128, init_ler=0.05, final_ler=0.005, brain="tka002"): dec = (final_ler / init_ler)**(1 / nb_epoch) traindata = DataManager(self.getDataFunc) for i in range(len(self.brains)): test = [self.brains[i]] train = self.brains[0:i] + self.brains[i + 1:len(self.brains)] if (brain != test[0]): continue print(train) print("Starting cv number", i, " - ", test[0], "out of", len(self.brains)) traindata.setTrain(train) cv_history = [] ler = init_ler start_time = time.time() for j in range(nb_epoch): print("Starting epoch:", j + 1, "/", nb_epoch) print("Genrating new training data") d = traindata.getData() sgd = SGD(lr=ler, decay=0, momentum=0.0, nesterov=False) self.model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) tr_h = self.model.fit(d[0], d[1], batch_size=batch_size, nb_epoch=1, verbose=2) print("train_loss", tr_h.history["loss"][0]) cv_history.append(tr_h.history["loss"][0]) ler *= dec elapsed_time = time.time() - start_time print("#################################################") print("\tTime training (", i, "):", elapsed_time) print("#################################################") self.model.save("../models/model_" + self.model_name + "_for_" + test[0] + ".mdl") with open( "../models/hist_" + self.model_name + "_for_" + test[0] + ".json", "w") as tf: tf.write(json.dumps(cv_history))
def handle_file_message(event): message_content = line_bot_api.get_message_content(event.message.id) with tempfile.NamedTemporaryFile(dir=static_tmp_path, prefix='file-', delete=False) as tf: for chunk in message_content.iter_content(): tf.write(chunk) tempfile_path = tf.name dist_path = tempfile_path + '-' + event.message.file_name dist_name = os.path.basename(dist_path) os.rename(tempfile_path, dist_path) line_bot_api.reply_message( event.reply_token, [ TextSendMessage(text='Save file.'), TextSendMessage(text=request.host_url + os.path.join('static', 'tmp', dist_name)) ])
def on_data(self, data): try: self.num_tweets += 1 if (self.num_tweets - 1 < self.wanted_tweets): with open(self.fetched_tweets_filename, 'a') as tf: json_load = json.loads(data) text = {'text': json_load['text']} print(text) tf.write(json.dumps(text)) tf.write('\n') return True else: return False except BaseException as be: print("Error on data: %s" % str(be)) return True
def run_synthesis(args, checkpoint_path, output_dir, sentences): metadata_filename = os.path.join(args.input_dir, 'train.txt') print(hparams_debug_string()) synth = Synthesizer() synth.load(checkpoint_path, gta=args.GTA) wav = load_wav(args.reference_audio) reference_mel = melspectrogram(wav).transpose() with open(metadata_filename, encoding='utf-8') as f: metadata = [line.strip().split('|') for line in f] frame_shift_ms = hparams.hop_size / hparams.sample_rate hours = sum([int(x[4]) for x in metadata]) * frame_shift_ms / (3600) print('Loaded metadata for {} examples ({:.2f} hours)'.format(len(metadata), hours)) if args.GTA==True: synth_dir = os.path.join(output_dir, 'gta') else: synth_dir = os.path.join(output_dir, 'natural') #Create output path if it doesn't exist os.makedirs(synth_dir, exist_ok=True) os.makedirs(os.path.join(synth_dir, 'wavs/'), exist_ok=True) print('starting synthesis') with open(os.path.join(synth_dir, 'map.txt'), 'w') as file: #for i, meta in enumerate(tqdm(metadata)): #text = meta[5] for i, text in enumerate(tqdm(sentences)): mel_output_filename = synth.synthesize(text=text, index=i+1, out_dir=synth_dir, log_dir=None, mel_filename=None, reference_mel=reference_mel) mels = np.load(mel_output_filename) wav = audio.inv_mel_spectrogram(mels.T) audio.save_wav(wav, os.path.join(synth_dir, 'wavs/speech-wav-{:05d}-mel.wav'.format(i+1))) with open(os.path.join(synth_dir, 'wavs/speech-wav-{:05d}.txt'.format(i+1)), 'w') as tf: tf.write(text) if hparams.predict_linear: # save wav (linear -> wav) wav = audio.inv_linear_spectrogram(linear.T) audio.save_wav(wav, os.path.join(synth_dir, 'wavs/speech-wav-{:05d}-linear.wav'.format(i+1))) #file.write('{}|{}|{}|{}\n'.format(text, mel_filename, mel_output_filename, wav_filename)) print('synthesized mel spectrograms at {}'.format(synth_dir))
def handle_message(event): global model if model is None: model = load_model("./static/linebot_model.h5") global graph with graph.as_default(): message_content = line_bot_api.get_message_content(event.message.id) with tempfile.NamedTemporaryFile(dir=static_tmp_path, prefix="jpg" + '-', delete=False) as tf: for chunk in message_content.iter_content(): tf.write(chunk) tempfile_path = tf.name #ファイルが保存された場所を示す dist_path = tempfile_path + '.' + "jpg" dist_name = os.path.basename(dist_path) #file名だけを抽出してくる os.rename(tempfile_path, dist_path) # ユーザから送信された画像のパスが格納されている filepath = os.path.join('./static', 'tmp', dist_name) class_label = [ "熊", "猫", "鹿", "犬", "象", "蛙", "キリン", "ゴリラ", "馬", "ライオン", "猿", "ウサギ", "アザラシ" ] img = image.load_img(filepath, target_size=(150, 150)) # 送信された画像を読み込み、リサイズする img = image.img_to_array(img) # 画像データをndarrayに変換する #img = np.array(img) #img = (img-img.mean()) / img.std() # モデルは4次元データを受け取るのでnp.array()にimgをリストとして渡し、4次元のデータにする data = np.array([img]) result = model.predict(data) predicted = result.argmax() pred_answer = "この写真は" + class_label[ predicted] + "に似ているね!また一つ動物博士に近づいたよ!" line_bot_api.reply_message(event.reply_token, TextSendMessage(text=pred_answer))
def promptuser(lines): """Display <lines> to user in their favourite editor. Then return the lines he entered ommiting empty lines and ones beginning with a #.""" tf = tempfile.NamedTemporaryFile(delete=False) for line in lines: tf.write(line + '\n') tf.close() editor = os.getenv('EDITOR') if editor == None: editor = 'vi' subp = subprocess.Popen([editor, tf.name]) subp.wait() f = open(tf.name) res = f.readlines() f.close() os.remove(tf.name) res = map(lambda x: x.strip(), res) #strip res = filter(None, res) #remove empty lines res = filter(lambda x: x[0] != '#', res) #remove comment lines return res
y_train, batch_size=batch_size, nb_epoch=1, verbose=2) print("train_loss", tr_h.history["loss"][0]) cv_history.append(tr_h.history["loss"][0]) ler *= dec if j % 50 == 0: final_model.save("../models/model_" + model_name + "_it_" + str(j) + "_cv_" + str(i) + ".mdl") final_model.save("../models/model_" + model_name + "_cv_" + str(i) + ".mdl") with open("hist_" + model_name + "_" + str(i) + ".json", "w") as tf: tf.write(json.dumps(cv_history)) quit() #model = load_model("../models/model_0.mdl") tr.reset() ### test stuff tt = imm.ImageManager() # load training data tt.init(test_brain) tt.createSlices(step=step + 1) tt.balance(bal_test) tt.split(1) # we will select the hole brain X_test_x = tt.getData(img_types, "2dx", inp_dim_2d)[0] y_test = X_test_x[1]
def start_learning(project_id, params=None): callback.stop_learning_tok = False res = requests.get(f'{base_url}/project/{project_id}/task/get', params=params, headers={'AUTH': get_auth_header()}) res_json = res.json() print(res_json) if (not (res_json['is_successful'])): return 'FAIL' occupy_task(project_id, {'task_index': res_json['task_index']}) model_url = res_json['model_url'] data_url = res_json['data_url'] label_url = res_json['label_url'] start_time = time.time() with TemporaryFile() as tf: tf.write(requests.get(url=model_url).content) _ = tf.seek(0) model = keras.models.load_model(h5py.File(tf, mode='r')) with TemporaryFile() as tf: tf.write(requests.get(url=data_url).content) _ = tf.seek(0) train_data = np.asarray(np.load(tf, allow_pickle=True)).astype(np.float32) with TemporaryFile() as tf: tf.write(requests.get(url=label_url).content) _ = tf.seek(0) train_label = np.asarray(np.load(tf, allow_pickle=True)).astype(np.float32) init_weight = get_weight(project_id) model.set_weights(init_weight.tolist()) task_index = int(res_json['task_index']) epoch = int(res_json['epoch']) batch_size = int(res_json['batch_size']) valid_rate = float(res_json['valid_rate']) print(train_data.shape) if (task_index == -1): validate(project_id) return 'STOP' try: model.fit(train_data, train_label, batch_size=batch_size, epochs=epoch, validation_split=valid_rate, callbacks=[callback], verbose=2) except ValueError as e: report_error(project_id=project_id, params={'error_message': e}) return 'ERROR' except TypeError as e: report_error(project_id=project_id, params={'error_message': e}) return 'ERROR' if callback.stop_learning_tok: return 'STOP' spent_time = time.time() - start_time with TemporaryFile() as tf: np.save( tf, np.array(model.get_weights(), dtype=object) - np.array(init_weight, dtype=object)) _ = tf.seek(0) update_success = update_learning(project_id=project_id, params={ 'task_index': res_json['task_index'], 'spent_time': spent_time }, gradient={'gradient': tf}) if (not (update_success)): return 'FAIL' return 'SUCCESS'
Tmat = TUM_vec_to_Tmat(R, t) # print('Tmat:',Tmat) if (file.split('.')[0] == '000000'): # print(type(tx)) tx = float(tx) ty = float(ty) tz = float(tz) qx = float(qx) qy = float(qy) qz = float(qz) qw = float(qw) # the first line tf.write(lines0) tf.write('\n') # the second line tf.write('%f %f %f %f %f %f %f %f\n' % (time, tx, ty, tz, qx, qy, qz, qw)) this_pose = Tmat else: this_pose = np.dot(Tmat, this_pose) # print('this_pose:',this_pose) tx = this_pose[0, 3] ty = this_pose[1, 3] tz = this_pose[2, 3] rot = this_pose[:3, :3] qw, qx, qy, qz = rot2quat(rot) tf.write('%f %f %f %f %f %f %f %f\n' %