def main(): args = parse_args() predictor = Predictor.create_predictor(args) args.model_type = args.model_type.lower() model_class, tokenizer_class = MODEL_CLASSES[args.model_type] tokenizer = tokenizer_class.from_pretrained( os.path.dirname(args.model_name_or_path)) if args.version_2_with_negative: raw_dataset = load_dataset('squad_v2', split='validation') else: raw_dataset = load_dataset('squad', split='validation') column_names = raw_dataset.column_names dataset = raw_dataset.map(partial(prepare_validation_features, tokenizer=tokenizer, args=args), batched=True, remove_columns=column_names, num_proc=4) batchify_fn = lambda samples, fn=Dict( { "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id), "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id) }): fn(samples) predictor = Predictor.create_predictor(args) predictor.predict(dataset, raw_dataset, args=args, collate_fn=batchify_fn)
def process_args(self, **args): if len(args) == 0: args = vars(parse_args()) for key, val in args.iteritems(): if val is None and hasattr(self, key): continue setattr(self, key, val) # Use +1 here just so we have some cushion trial_zeros = str( 1 + int(math.ceil(math.log(self.exp_trials) / math.log(10)))) trial_fmt = '%0' + trial_zeros + 'i' trial_str = trial_fmt % self.exp_trial self.output_file = results.ResultFile.filename_from_keys( agent=self.agent_type, size=self.domain_size, seed=self.domain_seed, complexity=self.domain_complexity, handicap=self.domain_handicap, trial=self.exp_trial) self.description = '{agent_type} agent, seed {seed}, handicap {handicap}, trial {trial}'.format( agent_type=self.agent_type, seed='%02i' % self.domain_seed, handicap='%02i' % self.domain_handicap, trial=trial_str) self.output_path = os.path.join(self.exp_directory, self.output_file) self._convert_handicap() self._process_gui_setting()
def main(): args = parse_args() env = SimpleEnv(n_bars = 2) agent = Agent( env, in_dim = len(env.get_state()), out_dim = env.action_space.n, learning_rate=LR, gamma=GAMMA) print("Optimizer", agent.optimizer) print("") print("Network", agent.model) print("") if args.mode == "train": train( env, agent, n_episodes=N_EPISODES, max_steps=MAX_STEPS, batch_size=BATCH_SIZE, eps_decay=EPS_DECAY, log=LOG, log_interval=LOG_INTERVAL if LOG_INTERVAL <= N_EPISODES else N_EPISODES, save_model_to=SAVE_MODEL_TO, load_model_from=args.checkpoint ) elif args.mode == "test": test(env, agent, checkpoint_path=args.checkpoint)
def main(): """Run the program. * Parse command-line arguments for output file name, config name etc. * Get colors scheme for each note from config file. * Initialize PyGame library. * Dump config if verbose is active. * Create an instance of the AnmaHandle class to handle everything. * Loop and get notes until user exits. * Write the resulting MIDI file. * Cleanup. """ arg = args.parse_args() colors = config.parse_colors(arg.conf) screen, player = config.init_pygame(arg.midi_output, arg.background) led_hndl = config.init_gpio() if arg.verbose: args.dump_config(arg) hndl = handle.AnmaHandle(arg, colors, screen, player, led_hndl) for i in range(60, 76): if '#' not in hndl.notes[i].name: hndl.notes[i].update_screen(hndl, (255, 255, 255)) for i in range(60, 76): if '#' in hndl.notes[i].name: hndl.notes[i].update_screen(hndl, (0, 0, 0)) if arg.verbose: for note in hndl.notes: print("{} {}".format(note.name, note.pitch)) while hndl.loop(): pass
def main(): args = parse_args() list = ["ci.jpg", "ci2.jpg", "ci3.jpg", "ci4.jpg"] calc = Calculator(args) print(calc.distance_matrix(list))
def main(): args, unknown = parse_args() signal.signal(signal.SIGINT, signal_handler) global launcher launcher = TmuxLauncher(args, unknown) launcher.run() signal.pause
def main(): args = parse_args() data_train, data_test, target_train, target_test = get_train_test_data( preprocessor=args.preprocessor, test_size=args.test_size) classifier = get_algorithm(args) model = classifier.fit(data_train, target_train) targets_predicted = model.predict(data_test) correct = 1 - np.mean(target_test != targets_predicted) print("Accuracy: {:.2%}".format(correct)) lib_classifier = get_library_version(args) lib_model = lib_classifier.fit(data_train, target_train) lib_targets_predicted = lib_model.predict(data_test) lib_correct = 1 - np.mean(target_test != lib_targets_predicted) print("Library Accuracy: {:.2%}".format(lib_correct)) print("Difference from Library version: {:+.2%}".format(correct - lib_correct)) # K-Fold Cross Validation data = get_data(args.preprocessor) k_fold = KFold(n_splits=len(data.data), shuffle=True, random_state=7) k_fold_accuracy = cross_val_score(lib_classifier, data.data, data.target, cv=k_fold, n_jobs=1, scoring='accuracy').mean() print("K-Fold Accuracy: {:.2%}".format(k_fold_accuracy))
def main(): args = parse_args() source_trn = open("/scratch/rjh347/input/source_trn.txt", "w") target_trn = open("/scratch/rjh347/input/target_trn.txt", "w") # creat train data -- simply split file1 into target / source with open(args.saveto + 'file1_s.txt', 'r') as train: for line in train: src, tgt = line.split(',') source_trn.write(src) target_trn.write(tgt) train.close() source_trn.close() target_trn.close() source_val = open("/scratch/rjh347/input/source_val.txt", "w") target_val = open("/scratch/rjh347/input/target_val.txt", "w") # creat evaluation data -- split file2 into target / source, save with 0.2 chance with open(args.saveto + 'file2_s.txt', 'r') as val: for line in val: src, tgt = line.split(',') thresh = random.randint(0, 4) if thresh > 3: source_val.write(src) target_val.write(tgt) val.close() source_val.close() target_val.close()
def infer(): args = parse_args() print(args) if args.use_gpu == 1: place = fluid.CUDAPlace(0) else: place = fluid.CPUPlace() inference_scope = fluid.Scope() test_files = [ os.path.join(args.test_data_dir, x) for x in os.listdir(args.test_data_dir) ] criteo_dataset = CriteoDataset() test_reader = paddle.batch(criteo_dataset.test(test_files), batch_size=args.batch_size) startup_program = fluid.framework.Program() test_program = fluid.framework.Program() cur_model_path = os.path.join(args.model_output_dir, 'epoch_' + args.test_epoch) with fluid.scope_guard(inference_scope): with fluid.framework.program_guard(test_program, startup_program): loss, auc, data_list, auc_states = eval('network_conf.' + args.model_name)( args.embedding_size, args.num_field, args.num_feat, args.layer_sizes_dnn, args.act, args.reg, args.layer_sizes_cin) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=data_list, place=place) fluid.io.load_persistables( executor=exe, dirname=cur_model_path, main_program=fluid.default_main_program()) for var in auc_states: # reset auc states set_zero(var.name, scope=inference_scope, place=place) loss_all = 0 num_ins = 0 for batch_id, data_test in enumerate(test_reader()): loss_val, auc_val = exe.run(test_program, feed=feeder.feed(data_test), fetch_list=[loss.name, auc.name]) num_ins += len(data_test) loss_all += loss_val * len(data_test) logger.info('TEST --> batch: {} loss: {} auc_val: {}'.format( batch_id + 1, loss_all / num_ins, auc_val)) print( 'The last log info is the total Logloss and AUC for all test data. ' )
def main(argv=None): ''' Main entry point ''' args = parse_args(argv) print(f'Running torch {torch.version.__version__}') profile_cuda_memory = args.config.cuda.profile_cuda_memory pin_memory = 'cuda' in args.device.type and not profile_cuda_memory dataloader = get_dataloader(args.config.data, args.seed_fn, pin_memory, args.num_devices, shuffle=args.shuffle) print(dataloader.dataset.stats) model = args.model(args.config.model, dataloader.dataset) action = args.action(args.action_config, model, dataloader, args.device) if args.action_type == 'train' and args.action_config.early_stopping: args.config.data.split = 'valid' args.config.data.max_examples = 0 action.validation_dataloader = get_dataloader(args.config.data, args.seed_fn, pin_memory, args.num_devices, shuffle=args.shuffle) if args.config.cuda.profile_cuda_memory: print('Profiling CUDA memory') memory_profiler = profile.CUDAMemoryProfiler( action.modules.values(), filename=profile_cuda_memory) sys.settrace(memory_profiler) threading.settrace(memory_profiler) step = 0 epoch = 0 if args.restore: restore_modules = { module_name: module for module_name, module in action.modules.items() if module_name not in args.reset_parameters } epoch, step = restore(args.restore, restore_modules, num_checkpoints=args.average_checkpoints, map_location=args.device.type, strict=not args.reset_parameters) model.reset_named_parameters(args.reset_parameters) if 'step' in args.reset_parameters: step = 0 epoch = 0 args.experiment.set_step(step) with ExitStack() as stack: stack.enter_context(profiler.emit_nvtx(args.config.cuda.profile_cuda)) stack.enter_context(set_detect_anomaly(args.detect_anomalies)) action(epoch, args.experiment, args.verbose)
def main(): args = parse_args() classifier = get_algorithm(args.algorithm) data_train, data_test, target_train, target_test = get_train_test_data(filename=args.file, test_size=args.test_size) model = classifier.fit(data_train, target_train) targets_predicted = model.predict(data_test) correct = 1 - np.mean(target_test != targets_predicted) print("Accuracy: {:.2%}".format(correct))
def main(): args = parse_args(sys.argv[1:]) create_config_dir() if args.load: walls = wallpaper.get_wallpapers(args.load) if walls: wallpaper.set_wallpapers(walls)
def main(): args = parse_args() if args.debug: t1 = time.time() if args.reduce_sci: reduce_science(args) if args.debug: t2 = time.time() print("Total Time taken: %0.2f s" %(t2-t1))
def main(): args = parse_args() # Generate base path: ".../$(args.output_dir)/run-$(date)-$(tag)" base_dir = generate_run_base_dir(args.model, args.tag, args.output_dir) log_file = os.path.join(base_dir, "log.txt") # Setup logging in base_dir/log.txt setup_logging(level=args.log_level, filename=log_file) # Wrap experiment in try-catch block to send error via email if it occurs try: # Run experiment experiment = Experiment(args, base_dir=base_dir) experiment.train() experiment.test() test_res_str = results_to_csv_str(experiment.test_result, experiment.test_result_pp) logger.info("Test results: ") logger.info(test_res_str) logger.info("Training took {}".format(experiment.training_time)) # Notify the user via e-mail and send the log file if args.notify is not None: subject = f"[MONOLAB {args.tag}] Training Finished!" message = ( f"The experiment in {base_dir} has finished training and " f"took {experiment.training_time}. Best loss: {experiment.best_val_loss}" f"Test results: \n" f"{ test_res_str }") notify_mail(address=args.notify, subject=subject, message=message, filename=log_file) except Exception as e: # Log error message tbstr = "".join(traceback.extract_tb(e.__traceback__).format()) errormsg = f"Traceback:\n{tbstr}\nError: {e}" logger.error(errormsg) # Notify exception if args.notify: subject = f"[MONOLAB {args.tag}] Training Error!" message = ( f"The experiment in {base_dir} has failed. An error occurred " f"during training:\n\n{errormsg}") notify_mail(address=args.notify, subject=subject, message=message, filename=log_file) raise e
def train(): args = parse_args() # add ce if args.enable_ce: SEED = 102 fluid.default_main_program().random_seed = SEED fluid.default_startup_program().random_seed = SEED print('---------- Configuration Arguments ----------') for key, value in args.__dict__.items(): print(key + ':' + str(value)) if not os.path.isdir(args.model_output_dir): os.mkdir(args.model_output_dir) loss, auc, data_list, auc_states = ctr_deepfm_model( args.embedding_size, args.num_field, args.num_feat, args.layer_sizes, args.act, args.reg) optimizer = fluid.optimizer.SGD( learning_rate=args.lr, regularization=fluid.regularizer.L2DecayRegularizer(args.reg)) optimizer.minimize(loss) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) dataset = fluid.DatasetFactory().create_dataset() dataset.set_use_var(data_list) pipe_command = 'python criteo_reader.py {}'.format(args.feat_dict) dataset.set_pipe_command(pipe_command) dataset.set_batch_size(args.batch_size) dataset.set_thread(args.num_thread) train_filelist = [ os.path.join(args.train_data_dir, x) for x in os.listdir(args.train_data_dir) ] print('---------------------------------------------') for epoch_id in range(args.num_epoch): start = time.time() dataset.set_filelist(train_filelist) exe.train_from_dataset( program=fluid.default_main_program(), dataset=dataset, fetch_list=[loss, auc], fetch_info=['epoch %d batch loss' % (epoch_id + 1), "auc"], print_period=1000, debug=False) model_dir = os.path.join(args.model_output_dir, 'epoch_' + str(epoch_id + 1)) sys.stderr.write('epoch%d is finished and takes %f s\n' % ((epoch_id + 1), time.time() - start)) fluid.io.save_persistables(executor=exe, dirname=model_dir, main_program=fluid.default_main_program())
def main(): args = parse_args() batch = load_data(args) count = 0 for item in batch: print(item) count += 1 if count > 10: break
def main(): # get command line parameters and lists of file specs arg_values = args.parse_args(sys.argv[1:]) cache_arg_files = args.glob_list_of_arg_values(arg_values.cache) no_cache_arg_files = args.glob_list_of_arg_values(arg_values.nocache) output_manifest_file_name = arg_values.output doc_root = None if not arg_values.doc_root else arg_values.doc_root[0] url_prefix = None if not arg_values.url_prefix else arg_values.url_prefix[0] force = arg_values.force # filter out files in cached list told to not cache filenames_to_cache = [ fn for fn in cache_arg_files if fn not in no_cache_arg_files ] # get a temporary file (fd, temp_file_name) = tempfile.mkstemp(prefix="temp_manifestgen_") os.close(fd) #convert file names on disk to urls cached_urls = convert_filenames_to_urls(filenames_to_cache, doc_root, url_prefix) # digest value of contents of all cached files content_md5_digest = digest_for_content(filenames_to_cache) # write temp manifest that reflects the current state write_cache_manifest(open(temp_file_name, "w"), cached_urls, arg_values.network, arg_values.fallback) write_name_value_comment(open(temp_file_name, "a"), "contentmd5", content_md5_digest) if force: write_name_value_comment(open(temp_file_name, "a"), "timestamp", str(datetime.datetime.now())) open(temp_file_name, "a").write("\n") # terminate last line #compare the newly created with the old and copy if it has changed digest_of_new_manifest = digest_for_cache_manifest(temp_file_name) digest_of_old_manifest = digest_for_cache_manifest( output_manifest_file_name) if digest_of_old_manifest != digest_of_new_manifest: print "digests differ, copying %s to %s" % (temp_file_name, output_manifest_file_name) open(output_manifest_file_name, "w").write(open(temp_file_name, "r").read()) else: print "cache manifest contents have not changed" # remove the temporary file if os.path.exists(temp_file_name): os.unlink(temp_file_name)
def main(): args = parse_args() predictor = Predictor.create_predictor(args) test_loader, src_vocab_size, tgt_vocab_size, bos_id, eos_id = create_infer_loader( args) _, vocab = IWSLT15.get_vocab() trg_idx2word = vocab.idx_to_token predictor.predict(test_loader, args.infer_output_file, trg_idx2word, bos_id, eos_id)
def main(): opts = args.parse_args() db = get_db(opts) importer = Importer(db, opts) if (not opts.skip_rankings and not opts.only_uservisits): importer.import_data_set('rankings') if (not opts.skip_uservisits or opts.only_uservisits): importer.import_data_set('uservisits') if (not opts.skip_crawls and not opts.only_uservisits): importer.import_data_set('crawl')
def main(): args = parse_args() predictor = Predictor.create_predictor(args) test_loader, src_vocab_size, tgt_vocab_size, bos_id, eos_id = create_infer_loader( args) tgt_vocab = Vocab.load_vocabulary(**test_loader.dataset.vocab_info['vi']) trg_idx2word = tgt_vocab.idx_to_token predictor.predict(test_loader, args.infer_output_file, trg_idx2word, bos_id, eos_id)
def main(): args = parse_args() testing_suite = BenchmarkSuite(args.capsule_dir, args.parallelism) results = testing_suite.test(args.num_samples) df = pd.DataFrame.from_records(results, columns=BenchmarkSuite.Result._fields) df.sort_values(by=list(df.columns), inplace=True, ignore_index=True) output.generate_output(output=df, csv_path=args.output_csv, graph_path=args.output_graph)
def train(): args = parse_args() # add ce if args.enable_ce: SEED = 102 fluid.default_main_program().random_seed = SEED fluid.default_startup_program().random_seed = SEED print(args) if not os.path.isdir(args.model_output_dir): os.mkdir(args.model_output_dir) loss, auc, data_list, auc_states = eval('network_conf.' + args.model_name)( args.embedding_size, args.num_field, args.num_feat, args.layer_sizes_dnn, args.act, args.reg, args.layer_sizes_cin) optimizer = fluid.optimizer.SGD( learning_rate=args.lr, regularization=fluid.regularizer.L2DecayRegularizer(args.reg)) optimizer.minimize(loss) dataset = fluid.DatasetFactory().create_dataset() dataset.set_use_var(data_list) dataset.set_pipe_command('python criteo_reader.py') dataset.set_batch_size(args.batch_size) dataset.set_filelist([ os.path.join(args.train_data_dir, x) for x in os.listdir(args.train_data_dir) ]) if args.use_gpu == 1: exe = fluid.Executor(fluid.CUDAPlace(0)) dataset.set_thread(1) else: exe = fluid.Executor(fluid.CPUPlace()) dataset.set_thread(args.num_thread) exe.run(fluid.default_startup_program()) for epoch_id in range(args.num_epoch): start = time.time() sys.stderr.write('\nepoch%d start ...\n' % (epoch_id + 1)) exe.train_from_dataset(program=fluid.default_main_program(), dataset=dataset, fetch_list=[loss, auc], fetch_info=['loss', 'auc'], debug=False, print_period=args.print_steps) model_dir = os.path.join(args.model_output_dir, 'epoch_' + str(epoch_id + 1), "checkpoint") sys.stderr.write('epoch%d is finished and takes %f s\n' % ((epoch_id + 1), time.time() - start)) fluid.io.save_persistables(fluid.default_main_program(), model_dir)
def train(): args = parse_args() if not os.path.isdir(args.model_output_dir): os.mkdir(args.model_output_dir) dense_input = fluid.layers.data(name="dense_input", shape=[dense_feature_dim], dtype='float32') sparse_input_ids = [ fluid.layers.data(name="C" + str(i), shape=[1], lod_level=1, dtype="int64") for i in range(1, 27) ] label = fluid.layers.data(name='label', shape=[1], dtype='int64') loss, auc_var, batch_auc_var = ctr_dnn_model_dataset( dense_input, sparse_input_ids, label, args.embedding_size, args.sparse_feature_dim) optimizer = fluid.optimizer.SGD(learning_rate=1e-4) optimizer.minimize(loss) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) dataset = fluid.DatasetFactory().create_dataset() dataset.set_use_var([dense_input] + sparse_input_ids + [label]) pipe_command = "python criteo_reader.py %d" % args.sparse_feature_dim dataset.set_pipe_command(pipe_command) dataset.set_batch_size(100) thread_num = 10 dataset.set_thread(thread_num) whole_filelist = [ "raw_data/part-%d" % x for x in range(len(os.listdir("raw_data"))) ] epochs = 20 for i in range(epochs): dataset.set_filelist(whole_filelist[:int(0.8 * len(whole_filelist))]) exe.train_from_dataset(program=fluid.default_main_program(), dataset=dataset, fetch_list=[auc_var], fetch_info=["auc"], debug=False) model_dir = args.model_output_dir + '/epoch' + str(i + 1) + ".model" sys.stderr.write("epoch%d finished" % (i + 1)) fluid.io.save_inference_model( model_dir, [dense_input.name] + [x.name for x in sparse_input_ids] + [label.name], [loss, auc_var], exe)
def run_cmdline_args(CONFIG_PATH): "Parse the command line arguments" global options options = parse_args(CONFIG_PATH) # Copy a default file if none exists if not path.isfile(options.filename): default = path.join(path.dirname(__file__), '../data/default_config.cfg') other = pkg_resources.resource_filename('poezio', 'default_config.cfg') if path.isfile(default): copy2(default, options.filename) elif path.isfile(other): copy2(other, options.filename) global firstrun firstrun = True
def test_game(): global game args = parse_args() game = Game(args) play_game(game.clone(), True) # timeit t = timeit.timeit("play_game(game.clone(), False)", globals=globals(), number=args.timeit_count) print("timeit : %s ms [%d] [total %s s]" % (round( (t / args.timeit_count) * 1000, 2), args.timeit_count, round(t, 2)))
async def test_mcts(): args = parse_args() game = Game(args) model = Model(game.state_shape(), game.action_size(), args) mcts = MCTS(model, args) experiences = await mcts.self_play(game) for result in experiences: print(result[2]) print("Game Score:", game.game_score()) for move in game.moves(): print(move)
def run_cmdline_args(CONFIG_PATH): "Parse the command line arguments" global options options = parse_args(CONFIG_PATH) # Copy a default file if none exists if not path.isfile(options.filename): default = path.join(path.dirname(__file__), '../data/default_config.cfg') other = path.join(path.dirname(__file__), 'default_config.cfg') if path.isfile(default): copy2(default, options.filename) elif path.isfile(other): copy2(other, options.filename) global firstrun firstrun = True
def main(): # get command line parameters and lists of file specs arg_values = args.parse_args(sys.argv[1:]) cache_arg_files = args.glob_list_of_arg_values(arg_values.cache) no_cache_arg_files = args.glob_list_of_arg_values(arg_values.nocache) output_manifest_file_name = arg_values.output doc_root = None if not arg_values.doc_root else arg_values.doc_root[0] url_prefix = None if not arg_values.url_prefix else arg_values.url_prefix[0] force = arg_values.force # filter out files in cached list told to not cache filenames_to_cache = [ fn for fn in cache_arg_files if fn not in no_cache_arg_files] # get a temporary file (fd, temp_file_name) = tempfile.mkstemp(prefix="temp_manifestgen_") os.close(fd) #convert file names on disk to urls cached_urls = convert_filenames_to_urls(filenames_to_cache, doc_root, url_prefix) # digest value of contents of all cached files content_md5_digest = digest_for_content(filenames_to_cache) # write temp manifest that reflects the current state write_cache_manifest(open(temp_file_name, "w"), cached_urls, arg_values.network, arg_values.fallback) write_name_value_comment(open(temp_file_name, "a"), "contentmd5", content_md5_digest) if force: write_name_value_comment(open(temp_file_name, "a"), "timestamp", str(datetime.datetime.now())) open(temp_file_name, "a").write("\n") # terminate last line #compare the newly created with the old and copy if it has changed digest_of_new_manifest = digest_for_cache_manifest(temp_file_name) digest_of_old_manifest = digest_for_cache_manifest(output_manifest_file_name) if digest_of_old_manifest != digest_of_new_manifest: print "digests differ, copying %s to %s"%(temp_file_name, output_manifest_file_name) open(output_manifest_file_name, "w").write(open(temp_file_name, "r").read()) else: print "cache manifest contents have not changed" # remove the temporary file if os.path.exists(temp_file_name): os.unlink(temp_file_name)
def main(): from args import parse_args args = parse_args() print(args) dataset = args.trainset + args.testset + args.devset print(dataset) for data_path in dataset: create_header_file(data_path) reader = MmapFile(data_path) # test sample line = reader.getvalue('10') try: sample = bytes.decode(line.strip()) print(sample) except: print(line)
def main(): args = parse_args() loaded_data = load_data() (X_train, y_train, X_test) = process(loaded_data) X = pd.concat([X_train, X_test]) trained_models = [] for target in TARGET_COLUMNS: model = make_simple_model(target) model = model.fit(X_train, y_train[target]) trained_models.append(model) y_fits = [] y_preds = [] for m in trained_models: result = m.predict(X) result_df = pd.DataFrame(result, index=X.index) y_fitted = result_df.loc[:"2020-04-30 23:30:00", :] y_pred = result_df.loc["2020-05-01 00:00:00":"2020-07-22 23:30:00", :] y_fits.append(y_fitted) y_preds.append(y_pred) y_fits_df = pd.concat(y_fits, axis=1) y_fits_df.columns = [f"{c}_fitted" for c in TARGET_COLUMNS] fits = pd.concat([y_fits_df, y_train.loc[:"2020-04-30 23:30:00":, :]], axis=1) sub = pd.concat(y_preds, axis=1) sub.columns = TARGET_COLUMNS sub.to_csv(f'submission.csv') if args.verbose: show_results(trained_models, fits) for trained_model in trained_models: print( pd.DataFrame(trained_model.cv_results_)[[ 'mean_train_score', 'std_train_score', 'mean_test_score', 'std_test_score' ]]) fits.plot() sub.plot() plt.show()
async def on_message(message): if message.content == '!stop' and message.author.guild_permissions.administrator: FileLogger.info('User requested shutdown') execute() await client.logout() return # we do not want the bot to reply to itself if message.author.bot: return if message.content.startswith('!') or message.content.startswith('!'): setup_guild_channel_list(message.author.guild) setup_guild_member_list(message.author.guild) user_auth = { 'guild_id': message.author.guild.id, 'user_id': message.author.id, 'user_admin': message.author.guild_permissions.administrator, 'channel_id': message.channel.id } content = message.content[1:] if message.attachments: content += f' {message.attachments[0].url}' msg = parse_args(user_auth, content) if msg: if isinstance(msg, Mapping): # it's a dict embed = get_embed(message.author, msg) await message.channel.send(embed=embed) elif isinstance(msg, list): # it's a list embed = embed_template(message.author) for i in range(len(msg)): embed.add_field(name=i, value=msg[i], inline=False) await message.channel.send(embed=embed) elif is_url(msg): # it's an url embed = Embed(color=0xffa200) embed.set_image(url=msg) await message.channel.send(embed=embed) else: await message.channel.send(msg)
def main(): args = parse_args() data_train, data_test, target_train, target_test = get_train_test_data( filename=args.file, test_size=args.test_size) classifier = get_algorithm(args) model = classifier.fit(data_train, target_train) targets_predicted = model.predict(data_test) correct = 1 - np.mean(target_test != targets_predicted) print("Accuracy: {:.2%}".format(correct)) lib_classifier = get_library_version(args) lib_model = lib_classifier.fit(data_train, target_train) lib_targets_predicted = lib_model.predict(data_test) lib_correct = 1 - np.mean(target_test != lib_targets_predicted) print("Library Accuracy: {:.2%}".format(lib_correct)) print("Difference from Library version: {:+.2%}".format(correct - lib_correct))
def main(): args = parse_args() if args.debug: t1 = time.time() if args.make_masterbias: make_masterbias(args) if args.make_masterdark: make_masterdark(args) if args.custom: args.reduce_twi = False args.reduce_sci = False custom(args) if args.reduce_twi: reduce_twighlight(args) if args.reduce_sci: reduce_science(args) if args.debug: t2=time.time() print("Total Time taken: %0.2f s" %(t2-t1))
def run_cmdline_args(CONFIG_PATH): "Parse the command line arguments" global options options = parse_args(CONFIG_PATH) # Copy a default file if none exists if not path.isfile(options.filename): default = path.join(path.dirname(__file__), '../data/default_config.cfg') other = pkg_resources.resource_filename('poezio', 'default_config.cfg') if path.isfile(default): copy2(default, options.filename) elif path.isfile(other): copy2(other, options.filename) # Inside the nixstore and possibly other distributions, the reference # file is readonly, so is the copy. # Make it writable by the user who just created it. if os.path.exists(options.filename): os.chmod(options.filename, os.stat(options.filename).st_mode | stat.S_IWUSR) global firstrun firstrun = True
from args import parse_args import server if __name__ == "__main__": args = parse_args() print "Starting cortext beacon" print " - Listening on port: " + str(args.port) print " ".join([" - Using", "_mocked_" if args.mock else "_real_", "data"]) if args.port < 1024: print "You selected a port < 1024, this requires special priveleges" print "Press Ctrl-C to shut down..." server.start_beacon_server(args.port, args.mock)
""" self.already_run.append((sync, None)) def can_sync_run(self, sync, args = None): """ Return False if a sync should not be run, True otherwise """ if sync.name in self.exclude: logger.debug('sync "%s" has been excluded', sync.name) return False if not self.allow_rerun and (sync, args) in self.already_run: logger.debug('sync "%s" (%s) has already been run', sync.name, args) return False return True # run! run! run! args = args.parse_args() if args.get('verbosity'): logging.basicConfig(level=logging.DEBUG) synctrex = Synctrex(**args) config = args.get('config') for config in utils.as_list(config): synctrex.load(config) synctrex.prepare() synctrex.run()
""" Set the value of an option temporarily """ try: RawConfigParser.set(self, section, option, value) except NoSectionError: pass firstrun = False # creates the configuration directory if it doesn't exist # and copy the default config in it CONFIG_HOME = environ.get("XDG_CONFIG_HOME") if not CONFIG_HOME: CONFIG_HOME = path.join(environ.get('HOME'), '.config') CONFIG_PATH = path.join(CONFIG_HOME, 'poezio') try: makedirs(CONFIG_PATH) except OSError: pass if not path.isfile(path.join(CONFIG_PATH, 'poezio.cfg')): copy2(path.join(path.dirname(__file__), '../data/default_config.cfg'), path.join(CONFIG_PATH, 'poezio.cfg')) firstrun = True options = parse_args(CONFIG_PATH) config = Config(options.filename) if firstrun: config.set('firstrun', True)
# Append any remaning files in the reindex queue to the rename queue # These are going on the end so we can let new_indexes have as much # room as it wants if reindex_queue: rename_queue.extend(new_indexes(last_index, None, reindex_queue)) process_rename_queue(syncdir, toread, rename_queue) def main(): 'Read the toread list' calibredb = CalibreDB() toread = ToRead(ARGS.toread, validator=calibredb.issue) syncdir = ExportDirectory(ARGS.syncdir) # Grab the ids of the first count entries wanted = toread.keys()[:ARGS.count] # Remove any files not in the list syncdir.keep_files(wanted) # Export any files not already present calibredb.export_files(wanted, syncdir) # Rename files so they sort in reading list order rename_files(syncdir, toread) if __name__ == '__main__': args.parse_args() logs.set_logging() main()