def modify_story(story_id): if request.method == 'GET': data = load_data() if data.get(story_id) is None: return redirect(url_for('add_story')) status = dict( zip(STATUSES, [x == data[story_id]['status'] for x in STATUSES])) return render_template('form.html', edit=True, story=data[story_id], status=status) elif request.method == 'POST': data = load_data() # update entry data[story_id] = { 'id': story_id, 'title': request.form.get('title'), 'description': request.form.get('description'), 'accept': request.form.get('accept'), 'business_value': int(request.form.get('business_value')), 'estimation': float(request.form.get('estimation')), 'status': request.form.get('status') } save_data(data) return redirect(url_for('list_stories'))
def add_story(): if request.method == 'GET': base_story = { 'id': None, 'title': '', 'description': '', 'accept': '', 'business_value': 1000, 'estimation': 2.5, 'status': 'Planning' } base_status = dict( zip(STATUSES, [x == base_story['status'] for x in STATUSES])) # return empty form return render_template('form.html', edit=False, story=base_story, status=base_status) elif request.method == 'POST': data = load_data() newuid = generate_uniqe_id(data) # add new entry data[newuid] = { 'id': newuid, 'title': request.form.get('title'), 'description': request.form.get('description'), 'accept': request.form.get('accept'), 'business_value': int(request.form.get('business_value')), 'estimation': float(request.form.get('estimation')), 'status': request.form.get('status') } save_data(data) # redirect to list page return redirect(url_for('list_stories'))
def get_data_to_prediction(self, code, num_step): dict_data_kospi = self.creon_7400_코스피() df_kospi = pd.DataFrame(dict_data_kospi) df_kospi = df_kospi.rename(columns={'close': 'close_kospi'}) dict_data_stock = self.creon_7400_주식차트조회(code, 'D', 1) df_stock = pd.DataFrame(dict_data_stock) dict_data_bond = self.creon_7818_국채() df_bond = pd.DataFrame(dict_data_bond) df_bond = df_bond.rename(columns={'close': 'close_bond'}) dict_data_7059 = self.creon_7059_주식지수(code) data = pd.merge(df_bond, df_kospi, "left", "date") data = pd.merge(data, df_stock, "left", "date") data['per'] = dict_data_7059['per'] data['pbr'] = dict_data_7059['pbr'] data['roe'] = dict_data_7059['roe'] data = data.sort_values(by='date', ascending=True).reset_index() windows = [5, 20, 60, 120] for window in windows: data['market_kospi_ma{}'.format(window)] = data['close_kospi'].rolling(window).mean() data['market_kospi_ma{}_ratio'.format(window)] = (data['close_kospi'] - data[ 'market_kospi_ma{}'.format(window)]) / \ data['market_kospi_ma{}'.format(window)] data['bond_k3y_ma{}'.format(window)] = data['close_bond'].rolling(window).mean() data['bond_k3y_ma{}_ratio'.format(window)] = (data['close_bond'] - data['bond_k3y_ma{}'.format(window)]) / \ data['bond_k3y_ma{}'.format(window)] data = data[['date', 'open', 'high', 'low', 'close', 'volume', 'per', 'pbr', 'roe', 'market_kospi_ma5_ratio', 'market_kospi_ma20_ratio', 'market_kospi_ma60_ratio', 'market_kospi_ma120_ratio', 'bond_k3y_ma5_ratio', 'bond_k3y_ma20_ratio', 'bond_k3y_ma60_ratio', 'bond_k3y_ma120_ratio']] date_start = str(data['date'].iloc[-num_step]) date_end = str(data['date'].iloc[-1]) data = data_manager.load_data(data, date_start, date_end, 'v2', False)[1] data = data.values.tolist() return [d + [0.5, 0.5] for d in data]
def main(args): """ Description: Create, train and test the conv net """ # Get the data train_data, train_labels, test_data, test_labels = load_data(args[1]) print(np.shape(train_data)) # Build our ConvNet conv_net = ConvolutionalNet(int(args[5]), int(args[6]), np.shape(train_data)[1], np.shape(test_data[0])[1]) # Train the ConvNet conv_net.train(train_data, train_labels, int(args[2]), int(args[3]), float(args[4])) # Test the ConvNet conv_net.test(test_data, test_labels)
import numpy as np import keras from keras.models import Sequential from keras.optimizers import SGD from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D from keras.layers import Dropout, Flatten, Dense, Activation from keras.layers.normalization import BatchNormalization from data_manager import RESIZE_DIMENSION, int2label, label2int, load_data NUM_CLASS = len(int2label) INPUT_DIM = RESIZE_DIMENSION * RESIZE_DIMENSION * 3 x_train, y_train = load_data("data/train", keep_original=True) y_train = keras.utils.to_categorical(y_train, num_classes=NUM_CLASS) x_test, y_test = load_data("data/train", keep_original=True) y_test = keras.utils.to_categorical(y_test, num_classes=NUM_CLASS) model = Sequential() model.add( BatchNormalization(input_shape=(RESIZE_DIMENSION, RESIZE_DIMENSION, 3))) model.add( Conv2D(filters=16, kernel_size=3, kernel_initializer='he_normal', activation='relu')) model.add(MaxPooling2D(pool_size=2)) model.add(BatchNormalization()) model.add( Conv2D(filters=32, kernel_size=3, kernel_initializer='he_normal',
img3d.shape, pixel_size_mm=PIXEL_SIZE_MM_SETUP, masks=cylindric_masks, radius_coef=radius_coefs[polimer_type]) dm.save_plot(fig, SAVE_IMG_DESKTOP_SETUP_FOLDER, f'hist {sample_id} {sample_name}') if __name__ == '__main__': polimer_type = ["PDL-05", "PDLG-5002"][0] radius_coefs = {"PDL-05": 0.9, "PDLG-5002": 0.95} paths = file_paths.get_benchtop_setup_paths(polimer_type) df = dm.load_data("setup_culindric_porosities.csv") #df = pd.DataFrame(columns = ['polimer_type', 'sample_number', 'date', 'mean', 'std']) for sample_id in range(len(paths)): sample_name = list(paths.keys())[sample_id] print( f"============== {sample_id} sample: {sample_name} ==============") print(sample_name.split()) img3d = ~get_bin_img(sample_name) print('tot: ', np.sum(img3d) / img3d.size) fig, ax = plt.subplots() ax.imshow(img3d[0], cmap="gray") dm.save_plot(fig, "previews", f'{sample_id} bin ' + sample_name)
def del_story(story_id): data = load_data() if data.get(story_id) is not None: del (data[story_id]) save_data(data) return redirect(url_for('list_stories'))
def executor(agrs): # Keras Backend 설정 if args.backend == 'tensorflow': os.environ['KERAS_BACKEND'] = 'tensorflow' elif args.backend == 'plaidml': os.environ['KERAS_BACKEND'] = 'plaidml.keras.backend' # 출력 경로 설정 output_path = os.path.join( settings.BASE_DIR, 'output/{}_{}_{}'.format(args.output_name, args.rl_method, args.net)) if not os.path.isdir(output_path): os.makedirs(output_path) # 파라미터 기록 with open(os.path.join(output_path, 'params.json'), 'w') as f: f.write(json.dumps(vars(args))) # 로그 기록 설정 file_handler = logging.FileHandler(filename=os.path.join( output_path, "{}.log".format(args.output_name)), encoding='utf-8') stream_handler = logging.StreamHandler(sys.stdout) file_handler.setLevel(logging.DEBUG) stream_handler.setLevel(logging.INFO) logging.basicConfig(format="%(message)s", handlers=[file_handler, stream_handler], level=logging.DEBUG) # 로그, Keras Backend 설정을 먼저하고 RLTrader 모듈들을 이후에 임포트해야 함 from agent import Agent from learners import DQNLearner # 모델 경로 준비 value_network_path = '' policy_network_path = '' if args.value_network_name is not None: value_network_path = os.path.join( settings.BASE_DIR, 'models/{}.h5'.format(args.value_network_name)) else: value_network_path = os.path.join( output_path, '{}_{}_value_{}.h5'.format(args.rl_method, args.net, args.output_name)) if args.policy_network_name is not None: policy_network_path = os.path.join( settings.BASE_DIR, 'models/{}.h5'.format(args.policy_network_name)) else: policy_network_path = os.path.join( output_path, '{}_{}_policy_{}.h5'.format(args.rl_method, args.net, args.output_name)) common_params = {} list_stock_code = [] list_chart_data = [] list_training_data = [] list_min_trading_unit = [] list_max_trading_unit = [] stock_code = args.stock_code get_data.get_data(stock_code, args.start_date, args.end_date, ver=args.ver) # 차트 데이터, 학습 데이터 준비 chart_data, training_data = data_manager.load_data(os.path.join( settings.BASE_DIR, 'data/{}/{}.csv'.format(args.ver, stock_code)), args.start_date, args.end_date, ver=args.ver) # 최소/최대 투자 단위 설정 min_trading_unit = max(int(10000 / chart_data.iloc[-1]['close']), 1) max_trading_unit = max(int(100000 / chart_data.iloc[-1]['close']), 1) # 공통 파라미터 설정 common_params = { 'rl_method': args.rl_method, 'delayed_reward_threshold': args.delayed_reward_threshold, 'net': args.net, 'num_steps': args.num_steps, 'lr': args.lr, 'output_path': output_path, 'reuse_models': args.reuse_models } # 강화학습 시작 learner = None common_params.update({ 'stock_code': stock_code, 'chart_data': chart_data, 'training_data': training_data, 'min_trading_unit': min_trading_unit, 'max_trading_unit': max_trading_unit }) learner = DQNLearner( **{ **common_params, 'value_network_path': value_network_path }) if learner is not None: pvs = learner.run(balance=args.balance, num_epoches=args.num_epoches, discount_factor=args.discount_factor, start_epsilon=args.start_epsilon, learning=args.learning) learner.save_models() return chart_data, pvs
policy_network_path = os.path.join( output_path, '{}_{}_policy_{}.h5'.format(args.rl_method, args.net, args.output_name)) common_params = {} list_stock_code = [] list_chart_data = [] list_training_data = [] list_min_trading_unit = [] list_max_trading_unit = [] for stock_code in args.stock_code: # 차트 데이터, 학습 데이터 준비 chart_data, training_data = data_manager.load_data(os.path.join( settings.BASE_DIR, 'data/{}/{}.csv'.format(args.ver, stock_code)), args.start_date, args.end_date, ver=args.ver) # 최소/최대 투자 단위 설정 min_trading_unit = max(int(100000 / chart_data.iloc[-1]['close']), 1) max_trading_unit = max(int(1000000 / chart_data.iloc[-1]['close']), 1) # 공통 파라미터 설정 common_params = { 'rl_method': args.rl_method, 'delayed_reward_threshold': args.delayed_reward_threshold, 'net': args.net, 'num_steps': args.num_steps, 'lr': args.lr, 'output_path': output_path,
d = { "costs": costs, "Y_prediction_test": Y_prediction_test, "Y_prediction_train": Y_prediction_train, "w": w, "b": b, "learning_rate": learning_rate, "num_iterations": num_iterations } return d if __name__ == "__main__": # load train data X_train, train_ids = load_data(sys.argv[1]) # load test data X_test, test_ids = load_data(sys.argv[2]) # preprocesing label data Y_train = ids2label(train_ids) Y_test = ids2label(test_ids) d = model(X_train, Y_train, X_test, Y_test, num_iterations=2000, learning_rate=0.005, print_cost=True)
def get_data(username): return jsonify(data_manager.load_data(username))
from data_manager import load_data, preprocess, get_train_data from environment import Environment ticker = 'MSFT' start_date = '2010-01-01' df = load_data(ticker, start_date) df['Date'] = df['Date'].astype('str') # print(df.head()) predf = preprocess(df) # print(predf.head()) print(Environment(predf).observe()) print(Environment(predf).get_price())
import data_manager as dm import neuron_analysis as ana import pupil_data as pup import matplotlib.pyplot as plt import numpy as np import pandas as pd import seaborn as sns #%% data = dm.load_data() spikes, pupil, contrast = dm.filter_data(data, session=21, areas=['all_vis'], time_bins=(50, 150)) results = ana.get_contrast_modulation(spikes, contrast) neuron_id = 8 sns.barplot(data=df2, x='contrast', y='fr') dm.survey_contrast_modulation(data, thresh_p=0.05, thresh_var=0.05) #%% pupil size # Look at a few random pupil traces spikes, pupil, contrast = dm.filter_data(data, session=21, areas=['all_vis'], time_bins=(25, 45)) n_rows = 4 n_cols = 6
def train(self, model): #training parameters batch_size = 128 maxepoches = self.maxepochs # learning_rate = 0.1 # lr_decay = 1e-6 # lr_drop = 20 # The data, shuffled and split between train and test sets: if self.data_name == "custom": assert (self.train_now == True) _, (x_test, y_test) = data_manager.load_data("mnist") (x_train, y_train) = self.in_memory_train_data if (len(np.shape(x_train)) == 3): x_train = tf.expand_dims(x_train, 3) if (len(y_train[0]) != self.num_classes): y_train = tensorflow.keras.utils.to_categorical( y_train, self.num_classes) else: (x_train, y_train), (x_test, y_test) = data_manager.load_data(self.data_name) x_train, x_test = self.normalize(x_train, x_test) # def lr_scheduler(epoch): # return learning_rate * (0.5 ** (epoch // lr_drop)) # reduce_lr = tensorflow.keras.callbacks.LearningRateScheduler(lr_scheduler) # #data augmentation # datagen = ImageDataGenerator( # featurewise_center=False, # set input mean to 0 over the dataset # samplewise_center=False, # set each sample mean to 0 # featurewise_std_normalization=False, # divide inputs by std of the dataset # samplewise_std_normalization=False, # divide each input by its std # zca_whitening=False, # apply ZCA whitening # rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180) # width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) # height_shift_range=0.1, # randomly shift images vertically (fraction of total height) # horizontal_flip=True, # randomly flip images # vertical_flip=False) # randomly flip images # # (std, mean, and principal components if ZCA whitening is applied). # datagen.fit(x_train) #optimization details # sgd = optimizers.SGD(lr=learning_rate, decay=lr_decay, momentum=0.9, nesterov=True) adam = optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['acc']) # early stopping estop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto') # training process in a for loop with learning rate drop every 25 epoches. historytemp = model.fit( x_train, y_train, shuffle=self.shuffle, batch_size=batch_size, # steps_per_epoch = np.shape(x_train)[0] // batch_size, epochs=maxepoches, validation_data=(x_test, y_test), callbacks=[estop], verbose=1) weights_path = "weights/mnistcnn_" + self.data_name + ".h5" model.save_weights(weights_path) return model
log = setup_logging("../output") # load the configuration log.info('configuration has been loaded') config=load_config(pathConfig) nb_clusters = config['nb_clusters'] random_state = config['random_state'] pathData = config['pathData'] features = config['features'] # load the data data = load_data(pathData, features) log.info('Data has been loaded') # Compute k-means clustering. results,model = clustering_KMeans(data, nb_clusters, random_state) log.info('the model has been trained') # add feature num cluster to the data frame features.append("num_cluster") clustered_data = pd.DataFrame(results,columns=features) # save data save_data(pathData,clustered_data) log.info('Results saved')
else: value_network_path = os.path.join( output_path, '{}_{}_value_{}.h5'.format(args.rl_method, args.net, output_name)) policy_network_path = os.path.join( output_path, '{}_{}_policy_{}.h5'.format(args.rl_method, args.net, output_name)) #################################################### 20210514 추가본, 각 주식 코드별로 구현 # 차트 데이터, 학습 데이터 준비 #chart_data, training_data = data_manager.load_data(os.path.join(settings.BASE_DIR,'data/{}/{}_data.txt'.format(args.ver, stock_code)), start_time, end_time, ver=args.ver) chart_data, training_data = data_manager.load_data( os.path.join( settings.BASE_DIR, 'files/OSSP_KOSPI/{}_day_data.txt'.format(stock_code)), ver=args.ver, start_time=start_time, end_time=end_time) # 최소/최대 투자 단위 설정 min_trading_unit = 1 #max(int((args.balance)/100 / chart_data.iloc[-1]['close']), 1) max_trading_unit = max( int(args.balance / chart_data.iloc[-1]['close']), 1) # 공통 파라미터 설정 common_params = { 'rl_method': args.rl_method, 'delayed_reward_threshold': args.delayed_reward_threshold, 'net': args.net, 'num_steps': args.num_steps, 'lr': args.lr,
env = Dummy() env.x0 = tf.placeholder(tf.float32, (batch_size, maxlen0, embedding_dim), name='x0') env.x1 = tf.placeholder(tf.float32, (batch_size, maxlen1, embedding_dim), name='x1') env.y = tf.placeholder(tf.float32, (batch_size, 1), name='y') build_model(env) sess = tf.InteractiveSession() sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) print('Load data start...') X_train_qu, X_train_col, y_train, X_test_qu, X_test_col, y_test, X_dev_qu, X_dev_col, y_dev = load_data( datapath='data/geo') print('Load data done...') if args.mode == 'train': train(sess, env, X_train_qu, X_train_col, y_train, X_dev_qu, X_dev_col, y_dev, epochs=train_epochs, load=False, shuffle=True, batch_size=batch_size,
def train(self, model): #training parameters batch_size = 128 maxepoches = self.maxepochs learning_rate = 0.1 lr_decay = 1e-6 lr_drop = 20 # The data, shuffled and split between train and test sets: (x_train, y_train), (x_test, y_test) = data_manager.load_data(self.data_name) x_train, x_test = self.normalize(x_train, x_test) def lr_scheduler(epoch): return learning_rate * (0.5**(epoch // lr_drop)) reduce_lr = tensorflow.keras.callbacks.LearningRateScheduler( lr_scheduler) # #data augmentation # datagen = ImageDataGenerator( # featurewise_center=False, # set input mean to 0 over the dataset # samplewise_center=False, # set each sample mean to 0 # featurewise_std_normalization=False, # divide inputs by std of the dataset # samplewise_std_normalization=False, # divide each input by its std # zca_whitening=False, # apply ZCA whitening # rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180) # width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) # height_shift_range=0.1, # randomly shift images vertically (fraction of total height) # horizontal_flip=True, # randomly flip images # vertical_flip=False) # randomly flip images # # (std, mean, and principal components if ZCA whitening is applied). # datagen.fit(x_train) #optimization details sgd = optimizers.SGD(lr=learning_rate, decay=lr_decay, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['acc']) # early stopping estop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto') # training process in a for loop with learning rate drop every 25 epoches. historytemp = model.fit( x_train, y_train, batch_size=batch_size, # steps_per_epoch = np.shape(x_train)[0] // batch_size, epochs=maxepoches, validation_data=(x_test, y_test), callbacks=[reduce_lr, estop], verbose=1) model.save_weights('weights/mnistcnn_test.h5') return model
def list_stories(): data = sorted(load_data().values(), key=lambda x: x['title']) return render_template('list.html', stories=data)
'8200445_PHAAET_rec290422019_PRincon_S1', '8200473_PHAAET_rec24052019_PRincon_S2', '8200487_PHAAET_rec04052019_PRincon_S1', '8200718_PHAAET_rec08032019_PRincon', '8201653_PHAAET_I.Cima_rec21012021_ninho 39_36_S1', '8201667_PHAAET_I.Cima_rec21012021_ninho 68_21_S1', '8201720_PHAAET_rec31122020_ICima_ninho 71_21_S1', '8201959_PHAAET_rec29122020_ICima_ninho 31_36_S1'] ### Detect events for a given datasets for filename in filenames: path = "" # Load data and filter acceleration signals with a butterworth filter initial_data = data_manager.load_data(filename, path) current_data = copy.deepcopy(initial_data) current_data.filter_accelerations(4, 0.4) all_data.append(current_data) print("Data loaded: "+filename) ''' ############################### ### Plot raw vs filtered signal fig, ax = plt.subplots(3,2,figsize = (8,6)) fig.suptitle("Raw vs filtered acceleration signals") ax[0,0].plot(initial_data.ax[10000:10200], 'b-') ax[0,0].set_ylim([-3, 3]) ax[0,1].plot(current_data.ax[10000:10200], 'b-') ax[0,1].set_ylim([-3, 3]) ax[0,0].set_ylabel("ax")
else: policy_network_path = os.path.join( output_path, '{}_{}_policy_{}.h5'.format(args.rl_method, args.net, args.output_name)) common_params = {} list_stock_code = [] list_chart_data = [] list_training_data = [] list_min_trading_unit = [] list_max_trading_unit = [] for stock_code in args.stock_code: # 데이터 읽어오기 DBdata = data_manager.read_DBdata(today) chart_data, training_data = data_manager.load_data(DBdata) # 최소/최대 투자 단위 설정 min_trading_unit = max(int(100000 / chart_data.iloc[-1]['close']), 1) max_trading_unit = max(int(1000000 / chart_data.iloc[-1]['close']), 1) # 공통 파라미터 설정 common_params = { 'rl_method': args.rl_method, 'delayed_reward_threshold': args.delayed_reward_threshold, 'net': args.net, 'num_steps': args.num_steps, 'lr': args.lr, 'output_path': output_path, 'reuse_models': args.reuse_models }