def main(): args = parse_args(should_use_model=False, should_use_rl=True) args.encoder_type = "identity" args.decoder_type = "identity" args.num_train_envs = 1 args.num_eval_envs = 1 ( vec_train_envs, vec_eval_envs, max_episode_steps, video_dir, model_dir, buffer_dir, video, device, replay_buffer, agent, L, ) = bootstrap_expert(args=args) train_agent( args=args, vec_train_envs=vec_train_envs, vec_eval_envs=vec_eval_envs, L=L, agent=agent, video=video, model_dir=model_dir, train_replay_buffer=replay_buffer, eval_replay_buffer=None, buffer_dir=buffer_dir, max_episode_steps=max_episode_steps, )
def initialize(): args = parse_args() torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) torch.backends.cudnn.benchmark = True #torch.backends.cudnn.deterministic = True assert len(args.learning_rates) == 1 or \ len(args.learning_rates) == len(args.lambda_values), \ 'Learning rates list should be length 1 or equal to lambda list length' if len(args.learning_rates) == 1: args.learning_rates *= len(args.lambda_values) args.lambda_values, args.learning_rates = zip( *sorted(zip(args.lambda_values, args.learning_rates))) assert (min(args.lambda_values) >= 0.0 and max(args.lambda_values) <= 1.0),\ 'Lambda values outside of [0, 1]' use_cuda = torch.cuda.is_available() and not args.cpu output_dir = os.path.join('results', args.output_directory, str(args.seed)) os.makedirs(output_dir, exist_ok=True) with open(os.path.join(output_dir, 'args.json'), 'w') as f: json.dump(vars(args), f, indent=2) if args.debug: logging.basicConfig(level=logging.DEBUG) return args, use_cuda, output_dir
def main(): args = parse_args(should_use_model=True, should_use_rl=True) args.load_model = True args.return_both_pixel_and_state = True args.change_angle = True args.num_train_envs = 2 args.num_eval_envs = 1 ( logbook, device, vec_train_envs, vec_eval_envs, state_space_obs, pixel_space_obs, action_size, train_replay_buffer, eval_replay_buffer, logging_dict, max_episode_steps, ) = bootstrap_envs_and_buffer(args) args.encoder_type = "identity" args.decoder_type = "identity" (video_dir, model_dir, buffer_dir, video, device, agent, L,) = bootstrap_agent( args, state_space_obs.shape, action_size, device ) model_dir, step = args.load_model_path.rsplit("_", 1) agent.load( model_dir=model_dir, step=step, ) start = time() # collect data across environments train_replay_buffer = utils.collect_both_state_and_obs_using_policy_vec( vec_env=vec_train_envs, num_samples=50000, replay_buffer=train_replay_buffer, policy=agent, save_video=args.save_video, ) train_replay_buffer.save(f"{args.save_buffer_path}/train") eval_replay_buffer = utils.collect_both_state_and_obs_using_policy_vec( vec_env=vec_eval_envs, num_samples=50000, replay_buffer=eval_replay_buffer, policy=agent, save_video=False, ) eval_replay_buffer.save(f"{args.save_buffer_path}/eval") end = time() print(f" Time to collect {args.num_samples} datapoints = {end - start}")
def main(): ret, root_path, old_version, new_version, token = argument_parser.parse_args() if ret == False: return tag_repos(root_path,new_version)
def main(): print( "Welcome to Group Finance. \n" "This client is continuously listening for commands. Input a command and press Enter.\n" "If you do not know the commands, check out README.md") while True: line = input() if line == 'exit': print("bye") break try: parse_args(line.split(" ")) except ArgumentParserError as err: print(err) print("")
def main(): ret, root_path, old_version, new_version, token = argument_parser.parse_args( ) if ret == False: return release(root_path, old_version, new_version, token)
def main(): ret, root_path, old_version, new_version, token = argument_parser.parse_args() if ret == False: return for project in projects: os.chdir(root_path) update_version(root_path, project, old_version[0], old_version[1], old_version[2], new_version[0], new_version[1], new_version[2])
def main(): # Receive the input arguments from command line. args = argument_parser.parse_args() # This path is command line input from the user. This is the root for the filetree of the processed images root_path_tuberculose_images_divided = args.path_target # This path is command line input from the user. This is where the downloaded raw images are stored right now image_origin = args.path_source # Define the paths for training, validating and testing train_paths = define_paths(path_base=root_path_tuberculose_images_divided, step="Train") valid_paths = define_paths(path_base=root_path_tuberculose_images_divided, step="Valid") test_paths = define_paths(path_base=root_path_tuberculose_images_divided, step="Test") # Create the folders for storing the training, validating and testing images create_folders(train_paths) create_folders(valid_paths) create_folders(test_paths) sick_healthy_dict = get_image_titles(image_origin) # Get the names of the images and convert the dict.keys object to a list dataset = list(sick_healthy_dict.keys()) # Set the size (as fraction) for the training set and the validation set. The remaining samples will be used for testing training_set_size = (1 / 2) validation_set_size = (1 / 4) # Calculate the absolute number of training and validation instances number_training_instances = calculate_number_of_instances(dataset, training_set_size) number_validation_instances = calculate_number_of_instances(dataset, validation_set_size) # Create the training, validation and testing sets training_data, dataset = get_random_instances(dataset, number_training_instances) validation_data, dataset = get_random_instances(dataset, number_validation_instances) testing_data = dataset # Create lists with the training, validating and testing labels training_labels_list = create_labels_list(training_data, sick_healthy_dict) validating_labels_list = create_labels_list(validation_data, sick_healthy_dict) testing_labels_list = create_labels_list(testing_data, sick_healthy_dict) write_images_to_folder(training_data, sick_healthy_dict, train_paths, image_origin) print("Done training images") write_images_to_folder(validation_data, sick_healthy_dict, valid_paths, image_origin) print("Done validating images") write_images_to_folder(testing_data, sick_healthy_dict, test_paths, image_origin) print("Done testing images")
import pandas as pd from argument_parser import parse_args import report_functions # parse command line args args = parse_args() with pd.ExcelWriter("res/" + "Report confronto " + args["name"] + ".xlsx") as excel_writer: # import files to compare: monitoring (status sheet) and rvtools (network) monitoring_file = pd.read_excel(args["monitoring"], sheet_name="Status") rvtools_file = pd.read_excel( args["rvtools"], sheet_name="vNetwork", na_values="unknown") rvtools_file = rvtools_file.dropna(subset=["IP Address"])[rvtools_file["Powerstate"] == "poweredOn"] # pick each vm with its ip address in rvtools rvtools_name_ip = dict(zip(list(rvtools_file["VM"]), [list(filter( lambda single_ip: ("." in single_ip), ip_list.replace(" ", "").split(",")))[0] for ip_list in list(rvtools_file["IP Address"])])) # pick each vm with its ip address in monitoring monitoring_name_ip = dict(zip(list(monitoring_file["Hostname"]), list(monitoring_file["IP"]))) # write not monitored vms onto the report report_functions.write_not_monitored_vms( rvtools_name2ip=rvtools_name_ip, monitoring_name2ip=monitoring_name_ip, excel_writer=excel_writer) # write vm in monitoring but not in rvtools
models = bootstrap_models_and_optimizers( args=args, obs_shape=obs_shape, action_size=action_size, device=device, logbook=logbook, ) # Train loop train_model( args=args, logbook=logbook, device=device, train_envs=vec_train_envs, eval_envs=vec_eval_envs, obs_shape=obs_shape, action_size=action_size, models=models, train_replay_buffer=train_replay_buffer, eval_replay_buffer=eval_replay_buffer, logging_dict=logging_dict, num_iters=args.num_iters, ) if __name__ == "__main__": args = argument_parser.parse_args(should_use_model=True, should_use_rl=False) main(args)
def main(): setup_requirements(base_dir) parsing_start = time.time() arg_dict = ap.parse_args() # Add to paths used during runtime for our rust libs append_to_environ("PATH", bin_dir) rust_ld_lib_addition = path.join(base_dir, ".compiled") if arg_dict["debug"]: rust_ld_lib_addition += ":" + path.join(base_dir, "src/func/target/debug/") rust_ld_lib_addition += ":" + path.join(base_dir, "target/debug/deps") # Set debug mode in case of a crash append_to_environ("RUST_BACKTRACE", "1") else: rust_ld_lib_addition += ":" + path.join(base_dir, "src/func/target/release/") rust_ld_lib_addition += ":" + path.join(base_dir, "target/release/deps") append_to_environ("LD_LIBRARY_PATH", rust_ld_lib_addition) # Grab input interval variables, use them for the function translation, # and write them out to a rust file # start_box, dimensions, variables = parse_box2(arg_dict["input"]) # import function_to_rust # tup = function_to_rust.translate(arg_dict["function"], variables) # (function, constants, part) = tup inputs = [tup[1] for tup in arg_dict['inputs']] inputs = "|".join(inputs) file_id = mk_file_hash(arg_dict["rust_function"]) function_filename = path.join( src_dir, "func/src/lib_generated_{}.rs".format(file_id)) if arg_dict["debug"]: executable = path.join(base_dir, 'target/debug/cooperative') else: executable = path.join(base_dir, 'target/release/cooperative') # Log output executable_args = [ '-c', arg_dict["constants"], '-f', arg_dict["interp_function"], '-i', inputs, "-x", str(arg_dict["input_epsilon"]), "-y", str(arg_dict["output_epsilon"]), "-S", "generated_" + file_id, # Function file suffix "-n", ",".join(b[0] for b in arg_dict["inputs"]), "-t", str(arg_dict["timeout"]), "-u", str(arg_dict["update"]), "-d" if arg_dict["debug"] else "", # If a debug run "-L" if arg_dict["logfile"] else "" ] iu.log(1, iu.cyan("Interpreted: ") + arg_dict["interp_function"]) iu.log(1, iu.cyan("Rust: ") + arg_dict["rust_function"]) iu.log(1, iu.cyan("Domain: ") + inputs) iu.log( 1, iu.cyan("Variables: ") + ", ".join(b[0] for b in arg_dict["inputs"])) iu.log(1, iu.cyan("Command: ") + ' '.join([executable] + executable_args)) parsing_end = time.time() # Use try so that we can catch control-c easily output = None logging = bool(arg_dict["logfile"]) log_file = arg_dict["logfile"] if type( arg_dict["logfile"]) is str else None try: with open(function_filename, 'w') as f: f.write(arg_dict["rust_function"]) if log_file: with open(log_file, 'w') as f2: f2.write("") f2.flush() start = time.time() term_time = None if arg_dict["timeout"] != 0: if arg_dict["grace"] == 0: term_time = start + arg_dict["timeout"] * 2 else: term_time = start + arg_dict["grace"] iu.log(1, iu.cyan("Running")) for line in iu.run_async(executable, executable_args, term_time): if logging and line.startswith("lb:"): # Hacky print(line.strip(), file=sys.stderr) if log_file: with open(log_file, 'a') as f2: f2.write(line.strip()) f2.write('\n') f2.flush() else: if arg_dict['dreal']: match = re.search("\[([^,]+),(.*)", line) if match: line = "[{},{}".format(-float(match.group(1)), match.group(2)) print(line.strip()) except KeyboardInterrupt: iu.warning("Caught ctrl-C, exiting now") finally: if not arg_dict["debug"]: os.remove(function_filename) try: os.remove( path.join(base_dir, ".compiled/libfunc_generated_" + file_id + ".so")) except: pass try: p = path.join( src_dir, "func/target/release/libfunc_generated_" + file_id + ".so") os.remove(p) except: pass try: p = path.join( src_dir, "func/target/release/func_generated_" + file_id + ".d") os.remove(p) except: pass end = time.time() if output: print(output) iu.log(0, iu.green("Parsing time: ") + str(parsing_end - parsing_start)) iu.log(0, iu.green("Solver time: ") + str(end - start))
import argument_parser from pprint import pprint args = argument_parser.parse_args() pprint(vars(args)) import os os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_HOME"] = "/nfs/xs/local/cuda-10.2" if len(args.gpu_ids) > 1: args.sync_bn = True from torch.utils.tensorboard import SummaryWriter from datasets.build_datasets import build_datasets from model.bisenet import BiSeNet from utils.saver import Saver from utils.trainer_kd import Trainer from utils.misc import AccCaches, get_curtime, print_model_parm_nums, load_state_dict import numpy as np def main(): # dataset trainset, valset, testset = build_datasets(args.dataset, args.base_size, args.crop_size) # 定义 student/teacher 模型
# /usr/bin/env python # TODO(John): Correct styling according to Google's Style Guide # TODO(John): Make this script importable (add a main method) # TODO(John): Figure out how to end script the correct way from subprocess import call import os import time from helpers import cd from argument_parser import parse_args # parse and store arguments arguments = parse_args() def download_sra(): # check if SRA directory exists, if not create it if not os.path.exists(arguments.dir + '/SRA'): print(time.strftime('[%a, %I:%M]'), 'Creating directory /SRA..') os.makedirs(arguments.dir + '/SRA') # move to newly created directory, download SRA files with cd(arguments.dir + '/SRA'): print(time.strftime("[%a, %I:%M]"), "Downloading SRA file: " + SRA\ + " to " + arguments.dir + '/SRA') # call fastq-dump using the SRA accension. if arguments.paired: call(["fastq-dump", "--split-files", SRA, "-gzip", ">",\ "../log/fastq_dump_log_{}.txt".\ format(time.strftime("[%a:%I:%M]"))]) else: call(["fastq-dump", SRA, "-gzip", ">",\
def main(): args = argument_parser.parse_args() print(args) torch.manual_seed(args.seed) torch.backends.cudnn.benchmark = True # hardcoding scannet # get handle to lmdb dataset lmdb_handle = dataset_base.LMDBHandle(os.path.join(constants.HDD_DATASET_ROOT, args.dataset, "dataset.lmdb"), args.memory_hog) # create train val and test sets train_set = get_active_dataset(args.active_selection_mode)(args.dataset, lmdb_handle, args.superpixel_dir, args.base_size, 'seedset_0') val_set = IndoorScenes(args.dataset, lmdb_handle, args.base_size, 'val') test_set = IndoorScenes(args.dataset, lmdb_handle, args.base_size, 'test') class_weights = None if args.use_balanced_weights: class_weights = calculate_weights_labels(get_active_dataset(args.active_selection_mode)(args.dataset, lmdb_handle, args.superpixel_dir, args.base_size, 'train')) saver = Saver(args) saver.save_experiment_config() summary = TensorboardSummary(saver.experiment_dir) writer = summary.create_summary() # get active selection method active_selector = get_active_selector(args, lmdb_handle, train_set) # for each active selection iteration for selection_iter in range(args.max_iterations): fraction_of_data_labeled = int(round(train_set.get_fraction_of_labeled_data() * 100)) if os.path.exists(os.path.join(constants.RUNS, args.dataset, args.checkname, f'runs_{fraction_of_data_labeled:03d}', "selections")): # resume: load selections if this is a rerun, and selections are available from a previous run train_set.load_selections(os.path.join(constants.RUNS, args.dataset, args.checkname, f'runs_{fraction_of_data_labeled:03d}', "selections")) elif os.path.exists(os.path.join(constants.RUNS, args.dataset, args.checkname, f'runs_{fraction_of_data_labeled:03d}', "selections.txt")): # resume: load selections if this is a rerun, and selections are available from a previous run train_set.load_selections(os.path.join(constants.RUNS, args.dataset, args.checkname, f'runs_{fraction_of_data_labeled:03d}', "selections.txt")) else: # active selection iteration train_set.make_dataset_multiple_of_batchsize(args.batch_size) # create model from scratch model = DeepLab(num_classes=train_set.num_classes, backbone=args.backbone, output_stride=args.out_stride, sync_bn=args.sync_bn, mc_dropout=((args.active_selection_mode.startswith('viewmc')) or(args.active_selection_mode.startswith('vote')) or args.view_entropy_mode == 'mc_dropout')) model = model.cuda() # create trainer trainer = Trainer(args, model, train_set, val_set, test_set, class_weights, Saver(args, suffix=f'runs_{fraction_of_data_labeled:03d}')) # train for args.epochs epochs lr_scheduler = trainer.lr_scheduler for epoch in range(args.epochs): trainer.training(epoch) if epoch % args.eval_interval == (args.eval_interval - 1): trainer.validation(epoch) if lr_scheduler: lr_scheduler.step() train_set.reset_dataset() epoch = trainer.load_best_checkpoint() # get best val miou / metrics _, best_mIoU, best_mIoU_20, best_Acc, best_Acc_class, best_FWIoU = trainer.validation(epoch, test=True) trainer.evaluator.dump_matrix(os.path.join(trainer.saver.experiment_dir, "confusion_matrix.npy")) writer.add_scalar('active_loop/mIoU', best_mIoU, train_set.get_fraction_of_labeled_data() * 100) writer.add_scalar('active_loop/mIoU_20', best_mIoU_20, train_set.get_fraction_of_labeled_data() * 100) writer.add_scalar('active_loop/Acc', best_Acc, train_set.get_fraction_of_labeled_data() * 100) writer.add_scalar('active_loop/Acc_class', best_Acc_class, train_set.get_fraction_of_labeled_data() * 100) writer.add_scalar('active_loop/fwIoU', best_FWIoU, train_set.get_fraction_of_labeled_data() * 100) # make active selection active_selector.select_next_batch(model, train_set, args.active_selection_size) # save selections trainer.saver.save_active_selections(train_set.get_selections(), args.active_selection_mode.endswith("_region")) trainer.train_writer.close() trainer.val_writer.close() print(selection_iter, " / Train-set length: ", len(train_set)) writer.close()
def main(): # script for training a model using 100% train set args = argument_parser.parse_args() print(args) torch.manual_seed(args.seed) lmdb_handle = dataset_base.LMDBHandle( os.path.join(constants.HDD_DATASET_ROOT, args.dataset, "dataset.lmdb"), args.memory_hog) train_set = IndoorScenes(args.dataset, lmdb_handle, args.base_size, 'train') val_set = IndoorScenes(args.dataset, lmdb_handle, args.base_size, 'val') test_set = IndoorScenes(args.dataset, lmdb_handle, args.base_size, 'test') train_set.make_dataset_multiple_of_batchsize(args.batch_size) model = DeepLab(num_classes=train_set.num_classes, backbone=args.backbone, output_stride=args.out_stride, sync_bn=args.sync_bn) model = model.cuda() class_weights = None if args.use_balanced_weights: class_weights = calculate_weights_labels(train_set) saver = Saver(args) trainer = Trainer(args, model, train_set, val_set, test_set, class_weights, Saver(args)) summary = TensorboardSummary(saver.experiment_dir) writer = summary.create_summary() start_epoch = 0 if args.resume: args.resume = os.path.join(constants.RUNS, args.dataset, args.resume, 'checkpoint.pth.tar') if not os.path.isfile(args.resume): raise RuntimeError(f"=> no checkpoint found at {args.resume}") checkpoint = torch.load(args.resume) start_epoch = checkpoint['epoch'] trainer.model.load_state_dict(checkpoint['state_dict']) trainer.optimizer.load_state_dict(checkpoint['optimizer']) trainer.best_pred = checkpoint['best_pred'] print( f'=> loaded checkpoint {args.resume} (epoch {checkpoint["epoch"]})' ) lr_scheduler = trainer.lr_scheduler for epoch in range(start_epoch, args.epochs): trainer.training(epoch) if epoch % args.eval_interval == (args.eval_interval - 1): trainer.validation(epoch) if lr_scheduler: lr_scheduler.step() epoch = trainer.load_best_checkpoint() _, best_mIoU, best_mIoU_20, best_Acc, best_Acc_class, best_FWIoU = trainer.validation( epoch, test=True) writer.add_scalar('test/mIoU', best_mIoU, epoch) writer.add_scalar('test/mIoU_20', best_mIoU_20, epoch) writer.add_scalar('test/Acc', best_Acc, epoch) writer.add_scalar('test/Acc_class', best_Acc_class, epoch) writer.add_scalar('test/fwIoU', best_FWIoU, epoch) trainer.train_writer.close() trainer.val_writer.close()