def main_menu(self): print("Main menu:") while True: print('Please select one of the following options:\n' '(R -> register, E -> exit, P -> print, F -> find)') user_input = input() if user_input == 'r' or user_input == 'R': std = self.register_student() self._students.append(std) print("Register a new student...") time.sleep(1) FileManager.write_file(r'files\students.txt', std) print("Done.") elif user_input == 'f' or user_input == 'F': self.find_student() elif user_input == 'p' or user_input == 'P': printer = Printer(self._students) # self.print_all_students() printer.show_printer_menu() printer.print_sorted_list(printer.get_user_input()) self.main_menu() else: print("Exiting program...") time.sleep(1) exit()
def fit(self): printer = Printer() while self.epoch < self.epochs: self.new_epoch() # increments self.epoch self.logger.set_mode("train") stats = self.train_epoch(self.epoch) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\ntrain: ") if self.epoch % self.test_every_n_epochs == 0 or self.phase1_will_end( ) or self.phase2_will_end(): self.logger.set_mode("test") stats = self.test_epoch(self.validdataloader) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\nvalid: ") if hasattr(self, "visdom"): self.visdom_log_test_run(stats) if hasattr(self, "visdom"): self.visdom.plot_epochs(self.logger.get_data()) self.check_events() # stores all stored values in the rootpath of the logger #if self.save_logger: self.logger.save() return self.logger.data
def compute_top_n_tokens_for_each_doc(top_n, first_id, last_id): models.connect_to_db(conf.DATABASE_FILENAME) cleaner = Cleaner() top_n_tokens_per_paper = {} for i in range(first_id, last_id + 1, increments): papers_to_process = ids_to_query(i, increments, last_id) for paper_id in papers_to_process: paper_query = models.Papers_NR.select().where( models.Papers.id == paper_id) if DEBUG: print(paper_query) print(len(paper_query)) if len(paper_query) > 0: paper_content = paper_query[0].paper_text pdf_name = paper_query[0].paper_name tokens = cleaner.tokenize(paper_content) token_frequencies = {} for token in tokens: if token not in token_frequencies: token_frequencies[token] = 1 else: token_frequencies[token] = token_frequencies[token] + 1 sorted_tokens = [(k, token_frequencies[k]) for k in sorted( token_frequencies, key=token_frequencies.get, reverse=True) ] top_n_tokens_per_paper[pdf_name] = sorted_tokens[:top_n] models.close_connection() printer = Printer() printer.print_dict(top_n_tokens_per_paper)
def _setup(self, options, sanitizer_name_short): self.__blacklist_file_path = os.path.join( options.output_root_path, self.__blacklists_dir_name, 'clang-' + sanitizer_name_short + self.__blacklist_file_ending) # dir_name.file_name.func_name self.__printer = Printer(options) self.__data = OrderedDict()
def setup(self, options): self.__printer = Printer(options) self.__analysing_funcs = { 'tsan': { 'data race': self.__tsan_analyse_data_race } }
def __init__(self): # Init related objects self.app = App(self) self.installer = Installer(self) self.local_op = LocalOperations() self.remote_op = RemoteOperations(self) self.printer = Printer() self.connect() self.setup()
def setup(self, options): self.__root_dir_path = os.path.join(options.output_root_path, self.__root_dir_name) utils.files.makedirs(self.__root_dir_path, True) self.__printer = Printer(options) self.__skeletons = {} self.__add_funcs = { 'tsan': { 'data race': self.__add_tsan_data_race } }
def setup(self, options): self.__printer = Printer(options) self.__duplicate_reports = [] self.__identifiers_funcs = { 'tsan': { 'data race': self.__tsan_data_race_identifiers, 'thread leak': self.__tsan_thread_leak_identifiers } } # TODO: split into separate lists for sanitizers and categories for better performance self.__known_identifiers = []
def __init__(self, options, sanitizer): self.__options = options self.__sanitizer = sanitizer self.__printer = Printer(options) # category.number self.__counters = {} self.__reports_dir_base_path = os.path.join(options.output_root_path, self.__reports_dir_name, sanitizer.name_short) self.__report_file = None self.__reports = []
def setup(self, options): self.__printer = Printer(options) self.__csv_base_dir_path = os.path.join(options.output_root_path, self.__csv_base_dir_name) utils.files.makedirs(self.__csv_base_dir_path) self.__controls = { 'tsan': { 'data race': { 'header_func': self.__header_tsan_data_race, 'process_func': self.__process_tsan_data_race } } }
def fit(self): printer = Printer() while self.epoch < self.epochs: self.new_epoch() # increments self.epoch self.logger.set_mode("train") stats = self.train_epoch(self.epoch) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\n" + self.traindataloader.dataset.partition + ": ") if self.epoch % self.test_every_n_epochs == 0 or self.epoch == 1: self.logger.set_mode("test") stats = self.test_epoch(self.validdataloader) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\n" + self.validdataloader.dataset.partition + ": ") if self.visdom is not None: self.visdom_log_test_run(stats) if self.visdom is not None: self.visdom.plot_epochs(self.logger.get_data()) if self.checkpoint_every_n_epochs % self.epoch == 0: print("Saving model to {}".format(self.get_model_name())) self.snapshot(self.get_model_name()) print("Saving log to {}".format(self.get_log_name())) self.logger.get_data().to_csv(self.get_log_name()) if self.epoch > self.early_stopping_smooth_period and self.check_for_early_stopping( smooth_period=self.early_stopping_smooth_period): print() print( f"Model did not improve in the last {self.early_stopping_smooth_period} epochs. stopping training..." ) print("Saving model to {}".format(self.get_model_name())) self.snapshot(self.get_model_name()) print("Saving log to {}".format(self.get_log_name())) self.logger.get_data().to_csv(self.get_log_name()) return self.logger return self.logger
def compute_document_frequencies(): models.connect_to_db(conf.DATABASE_FILENAME) first_id = 1 last_id_query = papers.select().order_by(papers.id.desc()).limit(1) last_id = last_id_query[0].id increments = 10 token_frequencies = {} for i in range(first_id, last_id + 1, increments): papers_to_process = ids_to_query(i, increments, last_id) for paper_id in papers_to_process: paper_query = papers.select().where(papers.id == paper_id) unique_tokens = set() if DEBUG: print(paper_query) print(len(paper_query)) if len(paper_query) > 0: paper_content = paper_query[0].paper_text paper_pdf_name = paper_query[0].pdf_name tokens = paper_content.strip().split() for token in tokens: #print(token) unique_tokens.add(token.lower()) for i, token in enumerate(unique_tokens): #print(token) if token not in token_frequencies: token_frequencies[token] = 1 else: token_frequencies[token] = token_frequencies[token] + 1 models.close_connection() sorted_tokens = [(k, token_frequencies[k]) for k in sorted(token_frequencies, key=token_frequencies.get)] printer = Printer() printer.print_token_frequency(sorted_tokens)
def compute_top_n_tokens_for_collection(top_n): models.connect_to_db(conf.DATABASE_FILENAME) first_id = 1 last_id_query = models.Papers_NR.select().order_by( models.Papers_NR.id.desc()).limit(1) last_id = last_id_query[0].id increments = 10 cleaner = Cleaner() token_frequencies = {} for i in range(first_id, last_id + 1, increments): papers_to_process = ids_to_query(i, increments, last_id) for paper_id in papers_to_process: paper_query = models.Papers.select().where( models.Papers.id == paper_id) if DEBUG: print(paper_query) print(len(paper_query)) if len(paper_query) > 0: paper_content = paper_query[0].paper_text paper_pdf_name = paper_query[0].pdf_name tokens = cleaner.tokenize(paper_content) for token in tokens: if token not in token_frequencies: token_frequencies[token] = 1 else: token_frequencies[token] = token_frequencies[token] + 1 models.close_connection() sorted_tokens = [(k, token_frequencies[k]) for k in sorted( token_frequencies, key=token_frequencies.get, reverse=True)] top_n_tokens = sorted_tokens[:top_n] printer = Printer() printer.print_token_frequency(top_n_tokens)
def fit(self): printer = Printer() while self.epoch < self.epochs: self.new_epoch() # increments self.epoch self.logger.set_mode("train") stats = self.train_epoch(self.epoch) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\ntrain: ") if self.epoch % self.test_every_n_epochs == 0 or self.phase1_will_end( ) or self.phase2_will_end(): self.logger.set_mode("test") stats = self.test_epoch(self.epoch) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\nvalid: ") self.visdom_log_test_run(stats) self.visdom.plot_epochs(self.logger.get_data()) self.check_events() return self.logger.data
def fit(self, epochs): printer = Printer() while self.epoch < epochs: self.new_epoch() # increments self.epoch self.logger.set_mode("train") stats = self.train_epoch(self.epoch) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\ntrain: ") if self.epoch % self.test_every_n_epochs == 0: self.logger.set_mode("test") stats = self.test_epoch(self.testdataloader) self.logger.log(stats, self.epoch) printer.print(stats, self.epoch, prefix="\ntest: ") if hasattr(self, "visdom"): self.visdom_log_test_run(stats) if hasattr(self, "visdom"): self.visdom.plot_epochs(self.logger.get_data()) self.logger.save() return self.logger.data
def setup(self, options): self.__printer = Printer(options) # sanitizer.category.count self.__data = {}
def execute(self, commands, depth): self.__hook.running() global_manage = Facade().get('global') component_manage = Facade().get('component') global_manage.if_turn_on() global_manage.depth = depth for command in commands: if isinstance(command, list): self.execute(command, depth + 1) global_manage.if_turn_on() while global_manage.is_loop: self.execute(command, depth + 1) global_manage.if_turn_on() global_manage.depth = depth continue component_name = command['component'].lower() component_args = command.get('args', {}) component_type = command.get('type', 'default') if global_manage.debug: Printer().init() Printer().add_row(['exec command', command]) Printer().add_row(['before_if_status', global_manage.is_if]) Printer().add_row(['before_break_status', global_manage.is_break]) Printer().add_row(['before_loop_status', global_manage.is_loop]) Printer().output() if command.get('db_args', None): component_args['db_args'] = global_manage.get(component_args['dbArgs'], '_db_args') global_manage.component_name = component_name global_manage.component_type = component_type component = component_manage.build(component_name, component_args) result = self.result_filter(component_args, component.run(component_type)) if command.get('return', None): global_manage.set(command['return'], result) if global_manage.debug: Printer().init() Printer().add_row(['after_if_status ', global_manage.is_if]) Printer().add_row(['after_break_status ', global_manage.is_break]) Printer().add_row(['after_loop_status ', global_manage.is_loop]) Printer().output() if component_name == 'if' and not global_manage.is_if: return if global_manage.is_break: return if not global_manage.is_loop: return
def main(): parser = argparse.ArgumentParser(description="PyTorch Training") parser.add_argument( '--net', type=str, default='deeplab', choices=['deeplabv3p', 'wdeeplabv3p', 'wsegnet', 'segnet', 'unet']) parser.add_argument('--p_dropout', type=str, default='0.5, 0.1', help='use which gpu to train, must be a \ comma-separated list of floats only') parser.add_argument('--parameter_per', type=str, default='0.5, 0.75', help='use which gpu to train, must be a \ comma-separated list of floats only') parser.add_argument('--wn', type=str, default='none', choices=[ 'none', 'haar', 'bior2.2', 'bior3.3', 'bior4.4', 'bior5.5', 'db2' ]) parser.add_argument('--backbone', type=str, default='resnet101', choices=['resnet50', 'resnet101', 'vgg16bn'], help='backbone name (default: resnet101)') parser.add_argument('--out_stride', type=int, default=16, help='network output stride (default: 8)') parser.add_argument('--dataset', type=str, default='pascal', choices=['pascal', 'coco', 'cityscapes'], help='dataset name (default: pascal)') parser.add_argument('--use_sbd', action='store_true', default=True, help='whether to use SBD dataset (default: True)') parser.add_argument( '--batch_average', action='store_true', default=True, help='whether to average the loss of the batch (default: True)') parser.add_argument('--workers', type=int, default=8, metavar='N', help='dataloader threads') parser.add_argument('--base_size', type=int, default=768, help='base image size') parser.add_argument('--crop_size', type=int, default=768, help='crop image size') parser.add_argument('--sync_bn', type=bool, default=None, help='whether to use sync bn (default: auto)') parser.add_argument( '--freeze_bn', type=bool, default=False, help='whether to freeze bn parameters (default: False)') parser.add_argument('--loss_type', type=str, default='ce', choices=['ce', 'focal'], help='loss func type (default: ce)') # training hyper params parser.add_argument('--epochs', type=int, default=None, metavar='N', help='number of epochs to train (default: auto)') parser.add_argument('--start_epoch', type=int, default=0, metavar='N', help='start epochs (default:0)') parser.add_argument('--batch_size', type=int, default=8, metavar='N', help='input batch size for \ training (default: auto)') parser.add_argument('--test_batch_size', type=int, default=4, metavar='N', help='input batch size for \ testing (default: auto)') parser.add_argument( '--use_balanced_weights', action='store_true', default=False, help='whether to use balanced weights (default: False)') # optimizer params parser.add_argument('--lr', '--learning_ratio', type=float, default=None, metavar='LR', help='learning rate (default: auto)') parser.add_argument('--lr_scheduler', type=str, default='poly', choices=['poly', 'step', 'cos'], help='lr scheduler mode: (default: poly)') parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='momentum (default: 0.9)') parser.add_argument('--weight_decay', type=float, default=5e-4, metavar='M', help='w-decay (default: 5e-4)') parser.add_argument('--nesterov', action='store_true', default=False, help='whether use nesterov (default: False)') # cuda, seed and logging parser.add_argument('--no_cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('-gpu', '--gpu_ids', type=str, default='0', help='use which gpu to train, must be a \ comma-separated list of integers only (default=0)') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') # checking point parser.add_argument('--resume', type=str, default=None, help='put the path to resuming file if needed') parser.add_argument('--checkname', type=str, default=None, help='set the checkpoint name') # finetuning pre-trained models parser.add_argument('--ft', '--fine_tuning', action='store_true', default=False, help='finetuning on a different dataset') # evaluation option parser.add_argument('--eval_interval', type=int, default=5, help='evaluuation interval (default: 1)') parser.add_argument('--no_val', action='store_true', default=False, help='skip validation during training') args = parser.parse_args() #在执行这条命令之前,所有命令行参数都给不会生效 args.cuda = not args.no_cuda and torch.cuda.is_available() if args.cuda: try: args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')] args.gpu_map = {} for index, gpu_id in enumerate(args.gpu_ids): args.gpu_map['cuda:{}'.format(gpu_id)] = 'cuda:{}'.format( index) args.gpu = list([i for i in range(len(args.gpu_ids))]) args.out_gpu = 0 except ValueError: raise ValueError( 'Argument --gpu_ids must be a comma-separated list of integers only' ) if args.sync_bn is None: if args.cuda and len(args.gpu_ids) > 1: args.sync_bn = True else: args.sync_bn = False args.p_dropout = tuple(float(s) for s in args.p_dropout.split(',')) args.parameter_per = tuple(float(s) for s in args.parameter_per.split(',')) # (0.5, 0.25) for voc in deeplabv3+ # (0.5, 0.25) or (0.5, 0.1) for voc in deeplabv3+wavelets # (0.25, 0.1) for cityscape in deeplabv3+ # (0, 0.1) for cityscape in deeplabv3+haar # default settings for epochs, batch_size and lr if args.epochs is None: epoches = { 'coco': 30, 'cityscapes': 200, 'pascal': 50, } args.epochs = epoches[args.dataset.lower()] args.weight_class = None if args.net == 'segnet' and args.dataset == 'pascal': args.weight_class = torch.tensor([ 0.1369, 1.4151, 3.0612, 1.5385, 1.7606, 1.7969, 0.7062, 1.0345, 0.5535, 1.699, 1.6026, 1.0081, 0.8621, 1.5306, 0.9167, 0.4124, 1.875, 1.506, 1.1029, 0.6452, 1.4286 ]).cuda() if args.batch_size is None: args.batch_size = 4 * len(args.gpu_ids) if args.test_batch_size is None: args.test_batch_size = args.batch_size if args.lr is None: lrs = { 'coco': 0.1, 'cityscapes': 0.01, 'pascal': 0.007, } #args.lr = lrs[args.dataset.lower()] / len(args.gpu_ids) args.lr = lrs[args.dataset.lower()] if args.checkname is None: args.checkname = args.net + '_' + str(args.backbone) + '_' + args.wn args.info_file = os.path.join( '.', 'info', args.dataset, args.net, args.checkname + '_' + datetime.now().strftime('%Y-%m-%d_%H-%M-%S.info')) args.weight_root = os.path.join( '.', 'weight', args.dataset, args.net, args.checkname + '_' + datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) my_mkdir(args.info_file, mode='file') my_mkdir(args.weight_root, mode='path') args.printer = Printer(args.info_file) torch.manual_seed(args.seed) trainer = Trainer(args) trainer.args.printer.pprint('Starting Epoch: {}'.format( trainer.args.start_epoch)) trainer.args.printer.pprint('Total Epoches: {}'.format( trainer.args.epochs)) args.time_begin = datetime.now() trainer.validation(epoch=trainer.args.start_epoch) for epoch in range(trainer.args.start_epoch, trainer.args.epochs): trainer.training(epoch) if not trainer.args.no_val and (epoch % args.eval_interval == (args.eval_interval - 1) or epoch >= (trainer.args.epochs - 10)): trainer.validation(epoch)
def __init__(self, options): super(TSanReportExtractor, self).__init__(options, Sanitizer('ThreadSanitizer', 'tsan')) self.__printer = Printer(options)
def __init__(self, options): self.__options = options self.__printer = Printer(options) self.__tasks = []
def setup(self, options): self.__printer = Printer(options) # sanitizer_name.category_name.(new|old) self.__data = OrderedDict()
def setup(self, options): self.__printer = Printer(options)