box_creator = PriorBoxCreator(model) prior_boxes = box_creator.create_boxes() layer_scale, box_arg = 0, 780 box_coordinates = prior_boxes[layer_scale][box_arg, :, :] image_path = '../images/' image_key = '007040.jpg' box_creator.draw_boxes(image_path + image_key, box_coordinates) data_path = '../datasets/VOCdevkit/VOC2007/' ground_truths = XMLParser(data_path + 'Annotations/').get_data() prior_box_manager = PriorBoxAssigner(prior_boxes, ground_truths) assigned_boxes = prior_box_manager.assign_boxes() prior_box_manager.draw_assigned_boxes(image_path, image_shape[0:2], image_key) batch_size = 7 train_keys, validation_keys = split_data(assigned_boxes, training_ratio=.8) assigned_image_generator = ImageGenerator(assigned_boxes, batch_size, image_shape[0:2], train_keys, validation_keys, data_path + 'JPEGImages/') transformed_image = next(assigned_image_generator.flow(mode='demo'))[0] transformed_image = np.squeeze(transformed_image[0]).astype('uint8') original_image = read_image(data_path + 'JPEGImages/' + validation_keys[0]) original_image = resize_image(original_image, image_shape[0:2]) plt.figure(1) plt.subplot(121) plt.title('Original image') plt.imshow(original_image) plt.subplot(122)
def main(**kwargs): import dill as pickle from datetime import datetime exp_dir = os.getcwd() + '/data/feature_net/' + kwargs['input_label'][0] + kwargs['output_label'][0] + '/' logger.configure(dir=exp_dir, format_strs=['stdout', 'log', 'csv'], snapshot_mode='last') json.dump(kwargs, open(exp_dir + '/params.json', 'w'), indent=2, sort_keys=True, cls=ClassEncoder) config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = kwargs.get('gpu_frac', 0.95) sess = tf.Session(config=config) mode = kwargs['mode'][0] visualize_training_data = kwargs['visualize_training_data'][0] visualize_testing_data = kwargs['visualize_testing_data'][0] visualize_new_data = kwargs['visualize_new_data'][0] if mode == 'restore': saver = tf.train.import_meta_graph(exp_dir + '-999.meta') saver.restore(sess, tf.train.latest_checkpoint(exp_dir)) graph = tf.get_default_graph() with sess.as_default() as sess: # folder = './data/policy/' + kwargs['env'][0] # buffer, fixed_num_of_contact = pickle.load(open('../saved/HandManipulateEgg-v0-fix9.pickle', 'rb')) buffer = {} name = 's1' paths, fixed_num_of_contact = pickle.load(open('../saved/soft/' + name + '80-dict.pickle', 'rb')) for key in paths: buffer[key] = paths[key] for name in ['s2', 's4', 's5', 's6', 'soft3']: paths, fixed_num_of_contact = pickle.load(open('../saved/soft/' + name + '80-dict.pickle', 'rb')) for key in paths: buffer[key] = np.concatenate([buffer[key], paths[key]], axis = 0) env = gym.make(kwargs['env'][0], obs_type = kwargs['obs_type'][0], fixed_num_of_contact = fixed_num_of_contact) for key in buffer: buffer[key] = buffer[key][:int(1e6)] niters = buffer['positions'].shape[0] // 100 print("total iteration: ", niters) ngeoms = env.sim.model.ngeom input_label = kwargs['input_label'][0] output_label = kwargs['output_label'][0] start = time.time() # paths = expand_data(buffer, ngeoms, fixed_num_of_contact, input_label, output_label) # print("expand data:", time.time() - start) paths = buffer start = time.time() train_data, test_data, vis_data, vis_data_test = split_data(paths, niters) print("split data:", time.time() - start) train_data['object_position'] = train_data['object_position'][:, :, :3] vis_data['original_object_position'] = vis_data['object_position'] vis_data_test['original_object_position'] = vis_data_test['object_position'] test_data['object_position'] = test_data['object_position'][:, :, :3] labels_to_dims = {} labels_to_dims['contacts'] = 3+6+ngeoms labels_to_dims['positions'] = 3 # labels_to_dims['object_position'] = 7 labels_to_dims['object_position'] = 3 labels_to_dims['joint_position'] = 24 labels_to_dims['object_vel'] = 6 labels_to_dims['joint_vel'] = 24 labels_to_dims['geoms'] = ngeoms dims = (labels_to_dims[input_label], labels_to_dims[output_label]) print("preparation done") num_episodes = 1 horizon = 100 if visualize_training_data: visualize_data(vis_data, env, fixed_num_of_contact, feature_net, mode, input_label) if visualize_testing_data: visualize_data(vis_data_test, env, fixed_num_of_contact, feature_net, mode, input_label)
y_true_classification = y_true[:, :, 4:(4 + num_classes)] return categorical_accuracy(y_true_classification, y_pred_classification) multibox_loss = MultiboxLoss(num_classes, neg_pos_ratio=2.0).compute_loss model.compile(optimizer=Adam(lr=3e-4), loss=multibox_loss, metrics=[class_accuracy]) box_creator = PriorBoxCreator(model) prior_boxes = box_creator.create_boxes() ground_truth_manager = XMLParser(ground_data_prefix, background_id=None, class_names=classes) ground_truth_data = ground_truth_manager.get_data() print('Number of ground truth samples:', len(ground_truth_data.keys())) train_keys, validation_keys = split_data(ground_truth_data, training_ratio=.8) prior_box_manager = PriorBoxManager(prior_boxes, box_scale_factors=[.1, .1, .2, .2], num_classes=num_classes) image_generator = ImageGenerator(ground_truth_data, prior_box_manager, batch_size, image_shape[0:2], train_keys, validation_keys, image_prefix, vertical_flip_probability=0, horizontal_flip_probability=0.5)
def main(config): raw_data = read_csv(config["filename"]) # Check if chps are already saved saved, chps = check_changepoints(config["filename"]) # Online changepoint if not saved: det = detection.BayesOnline() chp_online = det.find_changepoints(raw_data, past=50, prob_threshold=0.3) chps = chp_online[1:] # Evaluation bayesian analysis if config['split']: eval_bayesian(chps, raw_data) # Type of dataset (yearly,quarterly...) n_step = timeperiod(config['filename']) # Split in N train/test set (data + feature) if config['processing'] == 'indicators': train_data, test_data = split_with_indicators(raw_data, chps, n_step) elif config['processing'] == 'difference': raw_data = compute_diff(raw_data) train_data, test_data = split_data(raw_data, chps, n_step) else: raw_data = np.array(raw_data).reshape(-1, 1) train_data, test_data = split_data(raw_data, chps, n_step) # Cuda device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print(f"Device: {device}") print(torch.cuda.get_device_name(0)) # Setup the model input_size = train_data[0][0][0].size()[0] if config["regression"]: model = RegressionMLP(input_size=input_size) model = model.to(device) loss = nn.MSELoss() optimizer = torch.optim.Adadelta(model.parameters(), lr=config["lr"]) else: model = ClassficationMLP(input_size=input_size) model = model.to(device) loss = nn.BCELoss() optimizer = torch.optim.Adam(model.parameters(), lr=config["lr"]) epochs = config["epochs"] print(model) # Online training if config["online"]: global_writer = SummaryWriter( './runs/online/train/global/' + datetime.datetime.now().strftime('%m_%d_%H_%M')) for index, train_set in enumerate(train_data): print(f"----- DOMAIN {index} -----") train_loader = DataLoader(train_set, batch_size=config["batch_size"], shuffle=False) utils.training.train_online(train_loader, model=model, loss=loss, optimizer=optimizer, epochs=epochs, device=device, domain=index, global_writer=global_writer) # Test on domain just trained + old domains for past in range(index + 1): print(f"Testing model on domain {past}") test_loader = DataLoader(test_data[past], batch_size=1, shuffle=False) utils.training.test(model=model, loss=loss, test_loader=test_loader, device=device) # Continual learning with ER if config['continual']: utils.training.train_cl(train_set=train_data, test_set=test_data, model=model, loss=loss, optimizer=optimizer, config=config, device=device)
import tensorflow as tf from utils.utils import inputs, input_data, scale_data, split_data, predict_transf df = pd.read_csv('data/VCD_data_1.csv', parse_dates=[0]) model = tf.keras.models.load_model(os.path.join("temp_models/VCD-1.h5")) model.summary() app = flask.Flask(__name__, static_folder='static') data = scale_data(df) # print(data[-5:]) lookback = 3 # tf.saved_model.save(model, export_path) labels = ["Một ngày", "Hai ngày", "Ba ngày"] df['Date'] = df['Date'].apply(lambda x: (x.year, x.month, x.day)) date, train_x, test_x = split_data(df, lookback, 1) predict_data = predict_transf(model.predict(train_x)) @app.route("/", methods=['GET', 'POST']) def main(): values = zip(df['Date'], df['Vam co dong']) aut = zip(labels, df['Vam co dong'][-3:]) # print(aut) # if flask.request.method == 'GET': return (flask.render_template('index.html', predict_1="None", predict_2="None", predict_3="None", values=values,
from linear_regression import LinearRegression import sys sys.path.append('../') from utils import utils from loss import loss import numpy as np if __name__ == '__main__': with open('../data/housing.txt') as f: j = f.readlines() data = [map(float, s.split()) for s in j] data = np.array(data) X, y = data[:, :-1], data[:, -1] X_train, y_train, X_test, y_test = utils.split_data(X, y) X_train, X_test = utils.normalize(X_train), utils.normalize(X_test) lr = LinearRegression() lr.fit(X=X_train, y=y_train) y_train_pred, y_test_pred = lr.predict(X_train), lr.predict(X_test) train_abs_err = np.sum(np.abs(y_train_pred - y_train)) test_abs_err = np.sum(np.abs(y_test_pred - y_test)) print "train sae: {} , test sae: {}".format(train_abs_err, test_abs_err)
log_file_path = 'classification.log' # loading data data_manager = XMLParser(annotations_path) ground_truth_data = data_manager.get_data(['background', 'bottle']) print('Number of real samples:', len(ground_truth_data)) class_names = data_manager.class_names num_classes = len(class_names) print('Found classes: \n', class_names) # creating prior boxes prior_box_creator = PriorBoxCreator() prior_boxes = prior_box_creator.create_boxes() prior_box_manager = PriorBoxManager(prior_boxes, num_classes) train_keys, val_keys = split_data(ground_truth_data, validation_split) image_generator = ImageGenerator(ground_truth_data, prior_box_manager, batch_size, image_shape[0:2], train_keys, val_keys, image_prefix, vertical_flip_probability=0, suffix='') # model parameters sgd = SGD(lr=0.01, momentum=0.8, decay=0.1, nesterov=True) model = simple_CNN(image_shape, num_classes) model.compile(optimizer='adam', loss='categorical_crossentropy',
print("No preprocessed file found. Starting data preprocessing...") pairs = read_lines(os.path.join(start_root, DATA_DIR), FILENAME) pairs, path = preprocess_pipeline(pairs, cleaned_file, exp_contraction, max_len = max_sent_len) #data/prepro/eng-deu_cleaned_full.pkl ### Get sample ### print("Sample from data:") print(random.choice(pairs)) src_sents, trg_sents = [], [] if limit: pairs = pairs[:limit] print("Limit set: %s" % str(limit)) train_set, val_set, test_set = split_data(pairs, seed=args.seed) print("Data in train set:", len(train_set)) print("Data in val set:", len(val_set)) print("Data in test set:", len(test_set)) print("Building vocabularies...") if voc_all: train_data = train_set + val_set else: train_data = train_set src_sents = [item[0] for item in train_data] trg_sents = [item[1] for item in train_data] max_src_l = max_length(src_sents)
batch_size = 16 # Desired image dimensions img_width = 250 img_height = 80 # Mapping characters to integers char_to_num = layers.experimental.preprocessing.StringLookup( vocabulary=list(characters), num_oov_indices=0, mask_token=None) # Mapping integers back to original characters num_to_char = layers.experimental.preprocessing.StringLookup( vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True) # Splitting data into training and validation sets X_train, X_valid, y_train, y_valid = split_data(np.array(images), np.array(labels)) train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)) train_dataset = (train_dataset.map( lambda x, y: encode_single_sample(x, y, img_height, img_width, char_to_num ), num_parallel_calls=tf.data.experimental.AUTOTUNE).padded_batch( batch_size).prefetch(buffer_size=tf.data.experimental.AUTOTUNE)) validation_dataset = tf.data.Dataset.from_tensor_slices((X_valid, y_valid)) validation_dataset = (validation_dataset.map( lambda x, y: encode_single_sample(x, y, img_height, img_width, char_to_num ), num_parallel_calls=tf.data.experimental.AUTOTUNE).padded_batch( batch_size).prefetch(buffer_size=tf.data.experimental.AUTOTUNE))
type=float, help="Ratio of size of test dataset") args = parser.parse_args() # Using default netflix dataset, using url to download data or load data from path. if args.mode == "netflix": data = get_data() elif args.mode == "download": data = get_data(args.mode, args.data_url) else: data = get_data(args.mode, "", args.data_path) train, test = split_data(data, SEED, args.test_ratio) params = dict() params["epoch"] = args.epoch params["lambda"] = args.lamda params["momentum"] = args.momentum params["batch_size"] = args.batch_size params["lr"] = args.lr params["features"] = args.features params["users"] = np.unique(data[:, 0]).shape[0] params["products"] = np.unique(data[:, 1]).shape[0] if args.algorithm == "PMF": PMF_experiment = PMF(params) PMF_experiment.fit(train, test) PMF_experiment.plot_loss()