def train(self, token_lists, model_file): self.total = 0 self.correct = 0 self.incorrect = 0 self.top2 = 0 self.top3 = 0 self.prefix_model_file = "./trained_model/prefix_final/prefix.tfl" self.suffix_model_file = "./trained_model/suffix_final/suffix.tfl" self.prepare_data(token_lists) # xs = xs[:40960] # ys = ys[:40960] (pxs, pys, sxs, sys) = self.getTrainData(token_lists) with tf.Graph().as_default(): self.prefix_model = Models().create_network( len(self.string_to_number), len(self.string_to_number), model_name="bidirectional_attention_rnn", in_seq_len=self.in_seq_len, out_seq_len=self.out_seq_len, num_layers=2, memory_size=64, embedding_size=128, num_heads=8, scope="prefix") self.prefix_model.load(self.prefix_model_file) self.prefix_model.fit(pxs, pys, n_epoch=1, batch_size=512, shuffle=True, show_metric=False, run_id="Prefix Completion") self.prefix_model.save(self.prefix_model_file) with tf.Graph().as_default(): self.suffix_model = Models().create_network( len(self.string_to_number), len(self.string_to_number), model_name="bidirectional_attention_rnn", in_seq_len=self.in_seq_len, out_seq_len=self.out_seq_len, num_layers=2, memory_size=64, embedding_size=128, num_heads=8, scope="suffix") self.suffix_model.load(self.suffix_model_file) self.suffix_model.fit(sxs, sys, n_epoch=1, batch_size=512, shuffle=True, show_metric=False, run_id="Suffix Completion") self.suffix_model.save(self.suffix_model_file)
def cross_test(self, rfclf, feature_models, evalTag = True): #load data [tt_idx, x_te, y_te] = np.load(self.data_file_test) if self.model_dict['lrTModel']: [x_te_title, y_te] = np.load(self.data_file_title_test) if self.model_dict['scoreModel']: [x_te_score] = np.load(self.data_file_score_test) if self.model_dict['scoreTModel']: [x_te_score_title] = np.load(self.data_file_score_title_test) if self.model_dict['dictModel']: dic_result = np.array(np.load(self.dict_result)[0])[tt_idx] # dic_result[dic_result > 0] = 1 # dic_result[dic_result < 0] = -1 #get features n_features = [] for key in self.model_dict.keys(): if self.model_dict[key]: x_test = x_te if key == 'dictModel': n_features.append(dic_result) continue elif key == 'scoreModel': n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score)) continue elif key == 'scoreTModel': n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score_title)) continue elif key == 'lrTModel': x_test = x_te_title if key == 'lrModel' or key == 'lrTModel': n_features.append(feature_models[key].predict_proba(x_test)) else: n_features.append(feature_models[key].predict(x_test)) #start the second RF model x_second_te = np.column_stack(tuple(n_features)) rf_result = rfclf.predict(x_second_te) rf_result_proba = rfclf.predict_proba(x_second_te) #evaluate the precision of model rf_rp = np.copy(rf_result) rf_rp[rf_result_proba.max(axis=1) < 0.5] = 0 if evalTag: self.evaluate(rf_result, y_te) # self.DT.write_data('../data/negative.xls', self.origin_data_file, # tt_idx[rf_result == -1], y_te[rf_result == -1]) # self.DT.write_data('../data/positive.xls', self.origin_data_file, # tt_idx[rf_result == 1], y_te[rf_result == 1]) # self.DT.write_data('../data/zeros.xls', self.origin_data_file, # tt_idx[rf_result == 0], y_te[rf_result == 0]) return rf_result, y_te, rf_rp
def predict(self, pred_file = None): #load models rfclf = joblib.load(self.rfmodel_file) feature_models = dict() disable_models = ['dictModel', 'scoreModel', 'scoreTModel'] for key in self.model_dict.keys(): if self.model_dict[key] and (key not in disable_models): clf = joblib.load(self.model_dir + '/' + key) feature_models.setdefault(key, clf) if pred_file != None: test_data = self.pre_data_treate(pred_file) [x_te, x_te_title, x_te_score, x_te_score_title] = test_data #get features n_features = [] for key in self.model_dict.keys(): if self.model_dict[key]: x_test = x_te if key == 'scoreModel': n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score)) continue elif key == 'scoreTModel': n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score_title)) continue elif key == 'lrTModel': x_test = x_te_title if key == 'lrModel' or key == 'lrTModel': n_features.append(feature_models[key].predict_proba(x_test)) else: n_features.append(feature_models[key].predict(x_test)) #start the second RF model x_second_te = np.column_stack(tuple(n_features)) rf_result = rfclf.predict(x_second_te) rf_result_proba = rfclf.predict_proba(x_second_te) #evaluate the precision of model rf_rp = np.copy(rf_result) rf_rp[rf_result_proba.max(axis=1) < 0.5] = 0 self.DT.write_data('../data/negative.xls', pred_file, range(rf_result.shape[0]), rf_result) self.DT.write_data('../data/negative1.xls', pred_file, range(rf_result.shape[0]), rf_rp) else: return self.cross_test(rfclf, feature_models)
def initialise(self, param): self.parameters = param if not param.get('goal_selection_mode') == 'som': print('wrong goal selection mode, exit!') sys.exit(1) self.intrinsic_motivation = IntrinsicMotivation(param) if (self.parameters.get('train_cae_offline')) or ( self.parameters.get('train_som_offline')): self.models = Models(param, train_images=self.train_images) else: self.models = Models(param) plot_encoded = self.models.encoder.predict( np.asarray([self.test_images[0:5] ]).reshape(5, self.parameters.get('image_size'), self.parameters.get('image_size'), self.parameters.get('image_channels'))) plots.plots_cae_decoded( self.models.decoder, plot_encoded, self.test_images[0:5], image_size=self.parameters.get('image_size'), directory=self.parameters.get('directory_pretrained_models')) self.experiment_id = param.get('experiment_id') self.iteration = 0 self.pos = [] self.cmd = [] self.img = [] self.goal_code = [] self.current_goal_x = -1 self.current_goal_y = -1 self.current_goal_idx = -1 self.prev_goal_idx = -1 self.goal_image = np.zeros( (1, param.get('image_size'), param.get('image_size'), param.get('image_channels')), np.float32) np.random.seed() # change the seed self.prev_pos = self.get_starting_pos()
def train(self, token_lists, model_file): self.model_file = "./trained_model/random3/randomhole.tfl" self.prepare_data(token_lists) xs, ys = self.getTrainData(token_lists) with tf.Graph().as_default(): self.model = Models().create_network( self.in_max_int, self.out_max_int, model_name="bidirectional_attention_rnn", in_seq_len=self.in_seq_len, out_seq_len=self.out_seq_len, num_layers=2, memory_size=64, embedding_size=128, num_heads=6, scope="randomhole") self.model.load(self.model_file) self.model.fit(xs, ys, n_epoch=1, batch_size=256, shuffle=True, show_metric=False, run_id="Random Hole Completion") self.model.save(self.model_file)
def evaluate_model(trainX, trainy, testX, testy): epochs, batch_size = 15, 64 verbose, n_steps, n_length = 0, 4, 32 n_timesteps, n_features, n_outputs = trainX.shape[1], trainX.shape[ 2], trainy.shape[1] model = 'lstm' if model == 'cnnlstm': trainX = trainX.reshape( (trainX.shape[0], n_steps, n_length, n_features)) testX = testX.reshape((testX.shape[0], n_steps, n_length, n_features)) elif model == 'convlstm': trainX = trainX.reshape( (trainX.shape[0], n_steps, 1, n_length, n_features)) testX = testX.reshape( (testX.shape[0], n_steps, 1, n_length, n_features)) model = Models(model, n_timesteps, n_features, n_outputs, n_steps, n_length) model.model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size, verbose=verbose) _, accuracy = model.model.evaluate(testX, testy, batch_size=batch_size, verbose=verbose) print('Accuracy: {:.4f}'.format(accuracy))
def list_identifiers(status=[], first=0, max_size=0, order_by=None): db = get_db_instance() models = Models(db) model_identifiers = models.getIdentifiers() items = model_identifiers.all(status=status, skip=first, limit=max_size, order_by=('updated', 1)) i = 0 def show(x, alt, f=lambda a: a): return f(x) if x is not None else alt def isodate(t): return datetime.datetime.fromtimestamp(t).isoformat() for item in items: ean = show(item.ean, '-') asin = show(item.asin, '-') status = show(item.status, '-', str) updated = show(item.updated, '-', isodate) print( '#{i}\tasin:{asin}\tean:{ean}\tstatus:{status}\tupdated:{updated}'. format(i=i, asin=asin, ean=ean, status=status, updated=updated)) i += 1
def define_model(self, num_chan): models = Models(input_shape=(self.height, self.width, num_chan + 1*flag_expand_chan), classes=self.num_classes) if self.base_model == 'vgg16': models.vgg16() elif self.base_model == 'vgg19': models.vgg19() elif self.base_model == 'resnet50': models.resnet50() elif self.base_model == 'inceptionV3': models.inceptionV3() elif self.base_model == 'xception': models.xception() elif self.base_model == 'simple': models.simple() # TODO elif self.base_model == 'simple_resnet': models.simple_resnet() elif self.base_model == 'pspnet': models.simple_pspnet() elif self.base_model == 'simple_cascade_atrous': models.simple_cascade_atrous() elif self.base_model == 'simple_parallel_atrous': models.simple_parallel_atrous() else: print('Uknown base model') raise SystemExit # models.compile(optimizer=RMSprop(lr=1e-3)) # models.compile(optimizer=Adam()) # TODO sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True) models.compile(optimizer=sgd) self.model = models.get_model()
def initialise(self, param): self.parameters = param self.intrinsic_motivation = IntrinsicMotivation(param) self.models = Models(param) self.exp_iteration = param.get('exp_iteration') self.iteration = 0 self.pos = [] self.cmd = [] self.img = [] self.goal_code = [] self.current_goal_x = -1 self.current_goal_y = -1 self.current_goal_idx = -1 self.prev_goal_idx = -1 self.goal_image = np.zeros( (1, param.get('image_size'), param.get('image_size'), param.get('image_channels')), np.float32) np.random.seed() # change the seed self.prev_pos = self.get_starting_pos()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, default='configs/OT3+++R.yaml', help='Path to the config file.') parser.add_argument('--output_path', type=str, default='.', help="outputs path") parser.add_argument("--resume",default='True', action="store_true") #change to True is you need to retrain from pre-train model opts = parser.parse_args() cudnn.benchmark = True # Load experiment setting config = get_config(opts.config) # dataset set up dataset = My3DDataset(opts=config) train_loader = DataLoader(dataset=dataset, batch_size=config['batch_size'], shuffle=True, num_workers=config['nThreads']) config['vgg_model_path'] = opts.output_path trainer = Models(config) trainer.cuda() # Setup logger and output folders model_name = os.path.splitext(os.path.basename(opts.config))[0] output_directory = os.path.join(opts.output_path + "/outputs", model_name) train_writer = tensorboardX.SummaryWriter(os.path.join(opts.output_path + "/outputs/logs", model_name)) checkpoint_directory, image_directory = prepare_sub_folder(output_directory) shutil.copy(opts.config, os.path.join(output_directory, 'config.yaml')) # copy config file to output folder # Start training iterations = trainer.resume(checkpoint_directory, hyperparameters=config,need_opt=False) if opts.resume else 0 max_iter = int(config['n_ep']* len(dataset)/config['batch_size'])+1 while True: for it,out_data in enumerate(train_loader): for j in range(len(out_data)): out_data[j] = out_data[j].cuda().detach() if(config['models_name']=='dynamic_human' ): Xa_out, Xb_out, Yb_out, Xb_prev_out, Xb_next_out, Xa_mask, Yb_mask, rand_y_out, rand_y_mask=out_data trainer.update_learning_rate() with Timer("Elapsed time in update: %f"): # Main training code trainer.dynamic_gen_update(Xa_out, Xb_out, Yb_out,Xb_prev_out,Xb_next_out, Xa_mask, Yb_mask,rand_y_out, rand_y_mask, config) #torch.cuda.synchronize() # Dump training stats in log file if (iterations + 1) % config['log_iter'] == 0: print("Iteration: %08d/%08d" % (iterations + 1, max_iter)) write_loss(iterations, trainer, train_writer) if (iterations ) % config['image_display_iter'] == 0: write_image2display(iterations, trainer, train_writer) # Save network weights if (iterations+1 ) % config['snapshot_save_iter'] == 0: trainer.save(checkpoint_directory, iterations) iterations += 1 if iterations >= max_iter: sys.exit('Finish training')
def get_models(self): self.md = [] out = Outcome() for x in out.data: name = [c for c in x][0] #print(name) model = Models(self.datas['actual'], x[name]) self.md.append({'name': name, 'model': model})
def __init__(self, db_url, db_schema, **kwargs): self.engine = create_engine(db_url) Base.metadata.bind = self.engine DBSession.configure(bind=self.engine) self.models = Models(Base, db_schema) self.query = Query(self.models, DBSession) self.persen_denda = 'persen_denda' in kwargs and \ kwargs['persen_denda'] or 0
def delete(num): db = Models() if not db.view(): return render_template('success.html', not_delete=True) else: if request.method == 'GET': db.delete(str(num)) return render_template('success.html', delete=True)
def make_model(train_image, train_label, test_image, test_label, name): try: m = Models(shape=train_image.shape[1:], name=name) model = exec('m.' + bp.USE_MODEL + '(train_image, train_label, test_image, test_label)') print('info: input_shape:{}'.format(train_image.shape[1:])) except Exception as e: print(e) exit()
def main(args): pp = Preprocessing() # load data print("Loading Data.....\n\n") train_block, train_block_label = pp.read_train_file( args.train_data, args.train_label) test_block = pp.read_test_file(args.test_data) # explore data, do some visualization print("Exploring Data (see 'fig' folder for visualization) .....\n\n") viz = Visualization() # histogram for the lpc coefficient distribution viz.visualize_lpc_distribution(train_block) # histogram for the block length (or point of time) distribution viz.visualize_block_length_distribution(train_block) # plot one block of lpc coefficient for each speaker to look at the pattern of voice frequency viz.visualize_lpc_time_series(train_block) viz.visualize_fitted_lpc_series(train_block) max_length = 29 final_block_size = 18 print("Data Preprocessing (padding to fixed size blocks)....\n\n") # Take the best lengths (18), truncate the longer block, and pad the shorter block by the last row train_data = pp.pad_to_fixed_size_blocks(train_block, max_length, final_block_size) test_data = pp.pad_to_fixed_size_blocks(test_block, max_length, final_block_size) # dummy test label for convenience test_block_label = [[i] for i in np.zeros(len(test_data))] print("Generating Features (for ML Algorithms)... \n\n") # Generate fixed length feature vector for traditional machine learning input final_train_data = pp.convert_to_vectors(train_data, train_block_label, final_block_size) final_test_data = pp.convert_to_vectors(test_data, test_block_label, final_block_size) # See scatter plot to find out if there is grouping based on feature vector viz.lpc_scatter_plot(final_train_data) # Looks like there is a grouping, so let's try to classify using some popular algorithm model = Models() model.run_classification_models(final_train_data, final_test_data) print("SVM Prediction Saved (see 'results/submission.txt' )... \n\n") #Also try LSTM for classification model.run_LSTM_model(np.array(train_data), np.array(train_block_label), np.array(test_data)) print("LSTM Prediction Saved (see 'results/submission_lstm.txt' )... \n\n")
def main(): with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) import sklearn # eda = Eda() # cleaned_data = Cleaning() # features = Features() models = Models()
def creating_dataframe(self, dictionary): final_words = [] final_words1 = [] documents = [] l = [] z = [] docs = {} keys = dictionary.keys() for key in keys: kk = str(key) k = re.findall(r'\d{8}', kk) l.append(k) for i in l: for j in i: z.append(j) for key in z: # if key == '19234329': print( "###################### Generating topic labels for {} ############################" .format(key)) df = pd.DataFrame(dictionary[key]) df.columns = ['Text'] df_ = df['Text'].apply(lambda x: ''.join(x)) df_ = df_.str.lower() df_ = df_.apply(self.tokenize) df_ = df_.apply(self.replace) df_ = df_.apply(self.split) df_ = df_.apply(self.terms_only) df_ = df_.apply(lambda x: ' '.join(x)) df_ = df_.apply(lambda x: re.sub(r' +', ' ', x)) [final_words.append("".join(i).strip().split()) for i in df_] [final_words1.append(i) for i in final_words if len(i) >= 5] [ documents.append(re.sub(r' +', " ", (' '.join(i)))) for i in final_words1 ] if key in docs: docs[key].append(documents) else: docs[key] = documents mm = Models(5, 10, **docs) terms_to_wiki = mm.calling_methods('LDA') ll = Labels(terms_to_wiki) wiki_titles = ll.get_titles_wiki() equal_length = ll.remove_all_null_dicts_returned_from_wiki( **wiki_titles) frq = ll.calculating_word_frequency(**equal_length) results = ll.predicting_label(**frq) print(key, results) print('########### FINAL FILE EXECUTED ##################')
def _init_model(self): M = Models() model = M.FPN(img_ch=3, output_ch=1) # model = U_Net(img_ch=3, output_ch=1) if torch.cuda.device_count() > 1 and self.args.mgpu: print("Let's use", torch.cuda.device_count(), "GPUs!") # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs model = nn.DataParallel(model) self.model = model.to(self.device)
def start(self): bag_of_words, words = TermFrequency(self.trained).create_vocabulary() v = Vectorizer(self.trained, self.classify, words, bag_of_words) tfidf_trained = v.tfidf_for_tweets_trained evaluations = v.evaluations tfidf_to_classify = v.tfidf_for_tweets_to_classify models = Models(tfidf_trained, evaluations, tfidf_to_classify) prediction = models.svm_linear() return prediction
def load(self, token_lists, model_file): self.model_file = "./trained_model/random4/randomhole.tfl" self.prepare_data(token_lists) with tf.Graph().as_default(): self.model = Models().create_network(self.in_max_int, self.out_max_int, model_name="bidirectional_attention_rnn", in_seq_len=self.in_seq_len, out_seq_len=self.out_seq_len, num_layers=2, memory_size=128, embedding_size=128, num_heads=8, scope="randomhole") self.model.load(self.model_file)
def search(query): assert isinstance(query, str) # Model db = get_db_instance() models = Models(db) model_caches = models.getCaches() # Yahoo shopping API appid = get_yahoo_appid() assert appid is not None # Run y = YahooShopping(appid=appid, cache=model_caches) content = y.query(query=query, ignoreCache=False) print(content)
def train(): embeddings = np.load('text_embedding.npy', allow_pickle=True) sentiments = np.load('sentiments.npy', allow_pickle=True) texts = np.load('texts.npy', allow_pickle=True) all_texts = np.load('text_cache.npy', allow_pickle=True) categorical_sentiments = to_categorical(sentiments, num_classes=5) tokenizer = Tokenizer(num_words=300000, oov_token=None) tokenizer.fit_on_texts(all_texts) X_train, X_test, Y_train, Y_test = train_test_split(texts, categorical_sentiments, test_size=0.2) np.save("text_train.npy", X_train) np.save("sentiment_train.npy", Y_train) models = Models() logdir = "logs/scalars/" + datetime.now().strftime("%Y%m%d-%H%M%S") tensorboard_callback = TensorBoard(log_dir=logdir) models = [] bgruModel = "ensemble_bgru.h5" models.buil_pre_model(embeddings) model = models.model if os.path.isfile(filepath): model = load_model(filepath) checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min') callbacks_list = [checkpoint, tensorboard_callback] model.fit(pad_sequences(tokenizer.texts_to_sequences(X_train[:500000]), maxlen=75), Y_train[:500000], batch_size=512, epochs=50, validation_data=(pad_sequences(tokenizer.texts_to_sequences( X_test[:5000]), maxlen=75), Y_test[:5000]), callbacks=callbacks_list, shuffle=True) result = model.predict_on_batch( pad_sequences(tokenizer.texts_to_sequences([ " What happened 2 ur vegan food options?! At least say on ur site so i know I won't be able 2 eat anything for next 6 hrs #fail", " I sleep hungry and It gets harder everyday", "everything is great, i have lost some weight", "awesome, really cool", "should I play cards", "I am full and inshape", "is it okay to be that hungry at night?" ]), maxlen=75)) print("result: ", np.argmax(result, axis=-1), "\n")
def load(self, token_lists, model_file): self.total = 0 self.correct = 0 self.incorrect = 0 self.top2 = 0 self.top3 = 0 self.prefix_model_file = "./trained_model/prefix_final/prefix.tfl" self.suffix_model_file = "./trained_model/suffix_final/suffix.tfl" self.prepare_data(token_lists) with tf.Graph().as_default(): self.prefix_model = Models().create_network( len(self.string_to_number), len(self.string_to_number), model_name="bidirectional_attention_rnn", in_seq_len=self.in_seq_len, out_seq_len=self.out_seq_len, num_layers=2, memory_size=64, embedding_size=128, num_heads=8, scope="prefix") self.prefix_model.load(self.prefix_model_file) with tf.Graph().as_default(): self.suffix_model = Models().create_network( len(self.string_to_number), len(self.string_to_number), model_name="bidirectional_attention_rnn", in_seq_len=self.in_seq_len, out_seq_len=self.out_seq_len, num_layers=2, memory_size=64, embedding_size=128, num_heads=8, scope="suffix") self.suffix_model.load(self.suffix_model_file)
def __init__(self, logger=None): if logger is None: self.logger = logging.getLogger() else: self.logger = logger # Model db = get_db_instance() models = Models(db) model_caches = models.getCaches() # Proxy proxies = get_amazon_scraping_proxy() # Run self.amazon_scraper = AmazonScraper(cache=model_caches, proxies=proxies, logger=self.logger)
def update(num): form = NoteUpdate(request.form) # instantiating form to use the forms defined from the Form class in form.py if request.method == 'GET': return render_template('update.html', update=True, form=form, num=num) else: if not form.validate_on_submit(): # making sure that the form is validated before submission return render_template('update.html', update=True, not_validate=True, form=form, num=num) else: db = Models() name = request.form['note_name'] subject = request.form['note_subject'] content = request.form['note_content'] db.update(str(num), name, subject, content) return render_template('success.html', update=True, form=form)
def _init_model(self): criterion = nn.BCELoss() self.criterion = criterion.to(self.device) M = Models() model = M.PSP(img_ch=3, output_ch=1) self.model = model.to(self.device) # init_weights(self.model, 'kaiming', gain=1) # summary(self.model, input_size=(4, 448, 448)) self.model_optimizer = optim.Adamax(model.parameters(), lr=1e-3, weight_decay=0.01) self.scheduler = optim.lr_scheduler.CosineAnnealingLR( self.model_optimizer, T_max=len(self.train_queue))
def main(): parser = argparse.ArgumentParser() parser.add_argument( "-i", "--eval-dir", required=True, help="List of problem instances, i.e., [filename].txt files.") parser.add_argument( "-o", "--output-dir", required=True, help="List of truth files, one for each txt file in our --eval-dir.") args = parser.parse_args() if os.path.exists(args.output_dir): logging.warn( '[%s] already exists! Will possibly override files in it!' % args.output_dir) else: logging.debug('Creating [%s].' % args.output_dir) os.makedirs(args.output_dir) texts = load_texts(args.eval_dir) scalers = load_scalers() models = Models() splitted_texts = split_texts(texts) # NOTE using standard scaler splitted_texts_vectors = to_vectors(splitted_texts, scalers['standard'].transform) logging.debug('Making predictions...') for i, chunks_vectors_tuple in enumerate(splitted_texts_vectors, 1): chunks_vectors_pairs = itertools.combinations(chunks_vectors_tuple, 2) confidences = [0, 0] # (for false, for true) for a, b in chunks_vectors_pairs: pred_for_false, pred_for_true = models.classify_proba(a, b) # TODO if very confident about style change - break or something? confidences[0] += pred_for_false confidences[1] += pred_for_true y_pred = bool(np.argmax(confidences)) with open(os.path.join(args.output_dir, 'problem-%d.truth' % i), 'w') as f: json.dump({'changes': y_pred}, f) logging.debug('Done.')
def __init__(self): cudnn.benchmark = True # Load experiment setting config = get_config('configs/OT3.yaml') self.trainer = Models(config) self.trainer.cuda(config['cuda_device']) # Setup logger and output folders self.trainer.resume('outputs/OT3/checkpoints', hyperparameters=config, need_opt=False) self.trainer.eval() self.config = config self.dataset = My3DDataset(opts=self.config, is_Train=False) self.test_loader = DataLoader(dataset=self.dataset, batch_size=self.config['batch_size'] * 5, shuffle=False, num_workers=self.config['nThreads'])
def _init_dataset(self): M = Models() if self.args.mgpu: self.batch_size = 28 print('batch_size: ', self.batch_size) self.date = '/2020-05-06~11:38:23' self.Mo = M.FPN(img_ch=3, output_ch=1) else: self.batch_size = 7 print('batch_size: ', self.batch_size) self.date = '/2020-05-25~05:51:58' self.Mo = U_Net(img_ch=3, output_ch=1) test_images = Angioectasias(self.abnormality, mode='test') self.test_queue = DataLoader(test_images, batch_size=self.batch_size, drop_last=False)
def __init__(self, logger=None): if logger is None: self.logger = logging.getLogger() else: self.logger = logger # Model db = get_db_instance() models = Models(db) model_caches = models.getCaches() self.model_identifiers = models.getIdentifiers() # Yahoo yahoo_appid = get_yahoo_appid() self.yahoo_api = YahooShopping(appid=yahoo_appid, cache=model_caches, logger=self.logger) # Amazon proxies = get_amazon_scraping_proxy() self.amazon_api = AmazonScraper(cache=model_caches, proxies=proxies, logger=self.logger)