def main_menu(self):
print("Main menu:")
while True:
print('Please select one of the following options:\n'
'(R -> register, E -> exit, P -> print, F -> find)')
user_input = input()
if user_input == 'r' or user_input == 'R':
std = self.register_student()
self._students.append(std)
print("Register a new student...")
time.sleep(1)
FileManager.write_file(r'files\students.txt', std)
print("Done.")
elif user_input == 'f' or user_input == 'F':
self.find_student()
elif user_input == 'p' or user_input == 'P':
printer = Printer(self._students)
# self.print_all_students()
printer.show_printer_menu()
printer.print_sorted_list(printer.get_user_input())
self.main_menu()
else:
print("Exiting program...")
time.sleep(1)
exit()
def fit(self):
printer = Printer()
while self.epoch < self.epochs:
self.new_epoch() # increments self.epoch
self.logger.set_mode("train")
stats = self.train_epoch(self.epoch)
self.logger.log(stats, self.epoch)
printer.print(stats, self.epoch, prefix="\ntrain: ")
if self.epoch % self.test_every_n_epochs == 0 or self.phase1_will_end(
) or self.phase2_will_end():
self.logger.set_mode("test")
stats = self.test_epoch(self.validdataloader)
self.logger.log(stats, self.epoch)
printer.print(stats, self.epoch, prefix="\nvalid: ")
if hasattr(self, "visdom"): self.visdom_log_test_run(stats)
if hasattr(self, "visdom"):
self.visdom.plot_epochs(self.logger.get_data())
self.check_events()
# stores all stored values in the rootpath of the logger
#if self.save_logger:
self.logger.save()
return self.logger.data
def compute_top_n_tokens_for_each_doc(top_n, first_id, last_id):
models.connect_to_db(conf.DATABASE_FILENAME)
cleaner = Cleaner()
top_n_tokens_per_paper = {}
for i in range(first_id, last_id + 1, increments):
papers_to_process = ids_to_query(i, increments, last_id)
for paper_id in papers_to_process:
paper_query = models.Papers_NR.select().where(
models.Papers.id == paper_id)
if DEBUG:
print(paper_query)
print(len(paper_query))
if len(paper_query) > 0:
paper_content = paper_query[0].paper_text
pdf_name = paper_query[0].paper_name
tokens = cleaner.tokenize(paper_content)
token_frequencies = {}
for token in tokens:
if token not in token_frequencies:
token_frequencies[token] = 1
else:
token_frequencies[token] = token_frequencies[token] + 1
sorted_tokens = [(k, token_frequencies[k]) for k in sorted(
token_frequencies, key=token_frequencies.get, reverse=True)
]
top_n_tokens_per_paper[pdf_name] = sorted_tokens[:top_n]
models.close_connection()
printer = Printer()
printer.print_dict(top_n_tokens_per_paper)
def _setup(self, options, sanitizer_name_short):
self.__blacklist_file_path = os.path.join(
options.output_root_path, self.__blacklists_dir_name,
'clang-' + sanitizer_name_short + self.__blacklist_file_ending)
# dir_name.file_name.func_name
self.__printer = Printer(options)
self.__data = OrderedDict()
def setup(self, options):
self.__printer = Printer(options)
self.__analysing_funcs = {
'tsan': {
'data race': self.__tsan_analyse_data_race
}
}
def __init__(self):
# Init related objects
self.app = App(self)
self.installer = Installer(self)
self.local_op = LocalOperations()
self.remote_op = RemoteOperations(self)
self.printer = Printer()
self.connect()
self.setup()
def setup(self, options):
self.__root_dir_path = os.path.join(options.output_root_path, self.__root_dir_name)
utils.files.makedirs(self.__root_dir_path, True)
self.__printer = Printer(options)
self.__skeletons = {}
self.__add_funcs = {
'tsan': {
'data race': self.__add_tsan_data_race
}
}
def setup(self, options):
self.__printer = Printer(options)
self.__duplicate_reports = []
self.__identifiers_funcs = {
'tsan': {
'data race': self.__tsan_data_race_identifiers,
'thread leak': self.__tsan_thread_leak_identifiers
}
}
# TODO: split into separate lists for sanitizers and categories for better performance
self.__known_identifiers = []
def __init__(self, options, sanitizer):
self.__options = options
self.__sanitizer = sanitizer
self.__printer = Printer(options)
# category.number
self.__counters = {}
self.__reports_dir_base_path = os.path.join(options.output_root_path,
self.__reports_dir_name,
sanitizer.name_short)
self.__report_file = None
self.__reports = []
def setup(self, options):
self.__printer = Printer(options)
self.__csv_base_dir_path = os.path.join(options.output_root_path,
self.__csv_base_dir_name)
utils.files.makedirs(self.__csv_base_dir_path)
self.__controls = {
'tsan': {
'data race': {
'header_func': self.__header_tsan_data_race,
'process_func': self.__process_tsan_data_race
}
}
}
def fit(self):
printer = Printer()
while self.epoch < self.epochs:
self.new_epoch() # increments self.epoch
self.logger.set_mode("train")
stats = self.train_epoch(self.epoch)
self.logger.log(stats, self.epoch)
printer.print(stats,
self.epoch,
prefix="\n" +
self.traindataloader.dataset.partition + ": ")
if self.epoch % self.test_every_n_epochs == 0 or self.epoch == 1:
self.logger.set_mode("test")
stats = self.test_epoch(self.validdataloader)
self.logger.log(stats, self.epoch)
printer.print(stats,
self.epoch,
prefix="\n" +
self.validdataloader.dataset.partition + ": ")
if self.visdom is not None:
self.visdom_log_test_run(stats)
if self.visdom is not None:
self.visdom.plot_epochs(self.logger.get_data())
if self.checkpoint_every_n_epochs % self.epoch == 0:
print("Saving model to {}".format(self.get_model_name()))
self.snapshot(self.get_model_name())
print("Saving log to {}".format(self.get_log_name()))
self.logger.get_data().to_csv(self.get_log_name())
if self.epoch > self.early_stopping_smooth_period and self.check_for_early_stopping(
smooth_period=self.early_stopping_smooth_period):
print()
print(
f"Model did not improve in the last {self.early_stopping_smooth_period} epochs. stopping training..."
)
print("Saving model to {}".format(self.get_model_name()))
self.snapshot(self.get_model_name())
print("Saving log to {}".format(self.get_log_name()))
self.logger.get_data().to_csv(self.get_log_name())
return self.logger
return self.logger
def compute_document_frequencies():
models.connect_to_db(conf.DATABASE_FILENAME)
first_id = 1
last_id_query = papers.select().order_by(papers.id.desc()).limit(1)
last_id = last_id_query[0].id
increments = 10
token_frequencies = {}
for i in range(first_id, last_id + 1, increments):
papers_to_process = ids_to_query(i, increments, last_id)
for paper_id in papers_to_process:
paper_query = papers.select().where(papers.id == paper_id)
unique_tokens = set()
if DEBUG:
print(paper_query)
print(len(paper_query))
if len(paper_query) > 0:
paper_content = paper_query[0].paper_text
paper_pdf_name = paper_query[0].pdf_name
tokens = paper_content.strip().split()
for token in tokens:
#print(token)
unique_tokens.add(token.lower())
for i, token in enumerate(unique_tokens):
#print(token)
if token not in token_frequencies:
token_frequencies[token] = 1
else:
token_frequencies[token] = token_frequencies[token] + 1
models.close_connection()
sorted_tokens = [(k, token_frequencies[k]) for k in sorted(token_frequencies, key=token_frequencies.get)]
printer = Printer()
printer.print_token_frequency(sorted_tokens)
def compute_top_n_tokens_for_collection(top_n):
models.connect_to_db(conf.DATABASE_FILENAME)
first_id = 1
last_id_query = models.Papers_NR.select().order_by(
models.Papers_NR.id.desc()).limit(1)
last_id = last_id_query[0].id
increments = 10
cleaner = Cleaner()
token_frequencies = {}
for i in range(first_id, last_id + 1, increments):
papers_to_process = ids_to_query(i, increments, last_id)
for paper_id in papers_to_process:
paper_query = models.Papers.select().where(
models.Papers.id == paper_id)
if DEBUG:
print(paper_query)
print(len(paper_query))
if len(paper_query) > 0:
paper_content = paper_query[0].paper_text
paper_pdf_name = paper_query[0].pdf_name
tokens = cleaner.tokenize(paper_content)
for token in tokens:
if token not in token_frequencies:
token_frequencies[token] = 1
else:
token_frequencies[token] = token_frequencies[token] + 1
models.close_connection()
sorted_tokens = [(k, token_frequencies[k]) for k in sorted(
token_frequencies, key=token_frequencies.get, reverse=True)]
top_n_tokens = sorted_tokens[:top_n]
printer = Printer()
printer.print_token_frequency(top_n_tokens)
def fit(self):
printer = Printer()
while self.epoch < self.epochs:
self.new_epoch() # increments self.epoch
self.logger.set_mode("train")
stats = self.train_epoch(self.epoch)
self.logger.log(stats, self.epoch)
printer.print(stats, self.epoch, prefix="\ntrain: ")
if self.epoch % self.test_every_n_epochs == 0 or self.phase1_will_end(
) or self.phase2_will_end():
self.logger.set_mode("test")
stats = self.test_epoch(self.epoch)
self.logger.log(stats, self.epoch)
printer.print(stats, self.epoch, prefix="\nvalid: ")
self.visdom_log_test_run(stats)
self.visdom.plot_epochs(self.logger.get_data())
self.check_events()
return self.logger.data
def fit(self, epochs):
printer = Printer()
while self.epoch < epochs:
self.new_epoch() # increments self.epoch
self.logger.set_mode("train")
stats = self.train_epoch(self.epoch)
self.logger.log(stats, self.epoch)
printer.print(stats, self.epoch, prefix="\ntrain: ")
if self.epoch % self.test_every_n_epochs == 0:
self.logger.set_mode("test")
stats = self.test_epoch(self.testdataloader)
self.logger.log(stats, self.epoch)
printer.print(stats, self.epoch, prefix="\ntest: ")
if hasattr(self, "visdom"): self.visdom_log_test_run(stats)
if hasattr(self, "visdom"):
self.visdom.plot_epochs(self.logger.get_data())
self.logger.save()
return self.logger.data
def setup(self, options):
self.__printer = Printer(options)
# sanitizer.category.count
self.__data = {}
def execute(self, commands, depth):
self.__hook.running()
global_manage = Facade().get('global')
component_manage = Facade().get('component')
global_manage.if_turn_on()
global_manage.depth = depth
for command in commands:
if isinstance(command, list):
self.execute(command, depth + 1)
global_manage.if_turn_on()
while global_manage.is_loop:
self.execute(command, depth + 1)
global_manage.if_turn_on()
global_manage.depth = depth
continue
component_name = command['component'].lower()
component_args = command.get('args', {})
component_type = command.get('type', 'default')
if global_manage.debug:
Printer().init()
Printer().add_row(['exec command', command])
Printer().add_row(['before_if_status', global_manage.is_if])
Printer().add_row(['before_break_status', global_manage.is_break])
Printer().add_row(['before_loop_status', global_manage.is_loop])
Printer().output()
if command.get('db_args', None):
component_args['db_args'] = global_manage.get(component_args['dbArgs'], '_db_args')
global_manage.component_name = component_name
global_manage.component_type = component_type
component = component_manage.build(component_name, component_args)
result = self.result_filter(component_args, component.run(component_type))
if command.get('return', None):
global_manage.set(command['return'], result)
if global_manage.debug:
Printer().init()
Printer().add_row(['after_if_status ', global_manage.is_if])
Printer().add_row(['after_break_status ', global_manage.is_break])
Printer().add_row(['after_loop_status ', global_manage.is_loop])
Printer().output()
if component_name == 'if' and not global_manage.is_if:
return
if global_manage.is_break:
return
if not global_manage.is_loop:
return
def main():
parser = argparse.ArgumentParser(description="PyTorch Training")
parser.add_argument(
'--net',
type=str,
default='deeplab',
choices=['deeplabv3p', 'wdeeplabv3p', 'wsegnet', 'segnet', 'unet'])
parser.add_argument('--p_dropout',
type=str,
default='0.5, 0.1',
help='use which gpu to train, must be a \
comma-separated list of floats only')
parser.add_argument('--parameter_per',
type=str,
default='0.5, 0.75',
help='use which gpu to train, must be a \
comma-separated list of floats only')
parser.add_argument('--wn',
type=str,
default='none',
choices=[
'none', 'haar', 'bior2.2', 'bior3.3', 'bior4.4',
'bior5.5', 'db2'
])
parser.add_argument('--backbone',
type=str,
default='resnet101',
choices=['resnet50', 'resnet101', 'vgg16bn'],
help='backbone name (default: resnet101)')
parser.add_argument('--out_stride',
type=int,
default=16,
help='network output stride (default: 8)')
parser.add_argument('--dataset',
type=str,
default='pascal',
choices=['pascal', 'coco', 'cityscapes'],
help='dataset name (default: pascal)')
parser.add_argument('--use_sbd',
action='store_true',
default=True,
help='whether to use SBD dataset (default: True)')
parser.add_argument(
'--batch_average',
action='store_true',
default=True,
help='whether to average the loss of the batch (default: True)')
parser.add_argument('--workers',
type=int,
default=8,
metavar='N',
help='dataloader threads')
parser.add_argument('--base_size',
type=int,
default=768,
help='base image size')
parser.add_argument('--crop_size',
type=int,
default=768,
help='crop image size')
parser.add_argument('--sync_bn',
type=bool,
default=None,
help='whether to use sync bn (default: auto)')
parser.add_argument(
'--freeze_bn',
type=bool,
default=False,
help='whether to freeze bn parameters (default: False)')
parser.add_argument('--loss_type',
type=str,
default='ce',
choices=['ce', 'focal'],
help='loss func type (default: ce)')
# training hyper params
parser.add_argument('--epochs',
type=int,
default=None,
metavar='N',
help='number of epochs to train (default: auto)')
parser.add_argument('--start_epoch',
type=int,
default=0,
metavar='N',
help='start epochs (default:0)')
parser.add_argument('--batch_size',
type=int,
default=8,
metavar='N',
help='input batch size for \
training (default: auto)')
parser.add_argument('--test_batch_size',
type=int,
default=4,
metavar='N',
help='input batch size for \
testing (default: auto)')
parser.add_argument(
'--use_balanced_weights',
action='store_true',
default=False,
help='whether to use balanced weights (default: False)')
# optimizer params
parser.add_argument('--lr',
'--learning_ratio',
type=float,
default=None,
metavar='LR',
help='learning rate (default: auto)')
parser.add_argument('--lr_scheduler',
type=str,
default='poly',
choices=['poly', 'step', 'cos'],
help='lr scheduler mode: (default: poly)')
parser.add_argument('--momentum',
type=float,
default=0.9,
metavar='M',
help='momentum (default: 0.9)')
parser.add_argument('--weight_decay',
type=float,
default=5e-4,
metavar='M',
help='w-decay (default: 5e-4)')
parser.add_argument('--nesterov',
action='store_true',
default=False,
help='whether use nesterov (default: False)')
# cuda, seed and logging
parser.add_argument('--no_cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('-gpu',
'--gpu_ids',
type=str,
default='0',
help='use which gpu to train, must be a \
comma-separated list of integers only (default=0)')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
# checking point
parser.add_argument('--resume',
type=str,
default=None,
help='put the path to resuming file if needed')
parser.add_argument('--checkname',
type=str,
default=None,
help='set the checkpoint name')
# finetuning pre-trained models
parser.add_argument('--ft',
'--fine_tuning',
action='store_true',
default=False,
help='finetuning on a different dataset')
# evaluation option
parser.add_argument('--eval_interval',
type=int,
default=5,
help='evaluuation interval (default: 1)')
parser.add_argument('--no_val',
action='store_true',
default=False,
help='skip validation during training')
args = parser.parse_args() #在执行这条命令之前，所有命令行参数都给不会生效
args.cuda = not args.no_cuda and torch.cuda.is_available()
if args.cuda:
try:
args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')]
args.gpu_map = {}
for index, gpu_id in enumerate(args.gpu_ids):
args.gpu_map['cuda:{}'.format(gpu_id)] = 'cuda:{}'.format(
index)
args.gpu = list([i for i in range(len(args.gpu_ids))])
args.out_gpu = 0
except ValueError:
raise ValueError(
'Argument --gpu_ids must be a comma-separated list of integers only'
)
if args.sync_bn is None:
if args.cuda and len(args.gpu_ids) > 1:
args.sync_bn = True
else:
args.sync_bn = False
args.p_dropout = tuple(float(s) for s in args.p_dropout.split(','))
args.parameter_per = tuple(float(s) for s in args.parameter_per.split(','))
# (0.5, 0.25) for voc in deeplabv3+
# (0.5, 0.25) or (0.5, 0.1) for voc in deeplabv3+wavelets
# (0.25, 0.1) for cityscape in deeplabv3+
# (0, 0.1) for cityscape in deeplabv3+haar
# default settings for epochs, batch_size and lr
if args.epochs is None:
epoches = {
'coco': 30,
'cityscapes': 200,
'pascal': 50,
}
args.epochs = epoches[args.dataset.lower()]
args.weight_class = None
if args.net == 'segnet' and args.dataset == 'pascal':
args.weight_class = torch.tensor([
0.1369, 1.4151, 3.0612, 1.5385, 1.7606, 1.7969, 0.7062, 1.0345,
0.5535, 1.699, 1.6026, 1.0081, 0.8621, 1.5306, 0.9167, 0.4124,
1.875, 1.506, 1.1029, 0.6452, 1.4286
]).cuda()
if args.batch_size is None:
args.batch_size = 4 * len(args.gpu_ids)
if args.test_batch_size is None:
args.test_batch_size = args.batch_size
if args.lr is None:
lrs = {
'coco': 0.1,
'cityscapes': 0.01,
'pascal': 0.007,
}
#args.lr = lrs[args.dataset.lower()] / len(args.gpu_ids)
args.lr = lrs[args.dataset.lower()]
if args.checkname is None:
args.checkname = args.net + '_' + str(args.backbone) + '_' + args.wn
args.info_file = os.path.join(
'.', 'info', args.dataset, args.net, args.checkname + '_' +
datetime.now().strftime('%Y-%m-%d_%H-%M-%S.info'))
args.weight_root = os.path.join(
'.', 'weight', args.dataset, args.net,
args.checkname + '_' + datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
my_mkdir(args.info_file, mode='file')
my_mkdir(args.weight_root, mode='path')
args.printer = Printer(args.info_file)
torch.manual_seed(args.seed)
trainer = Trainer(args)
trainer.args.printer.pprint('Starting Epoch: {}'.format(
trainer.args.start_epoch))
trainer.args.printer.pprint('Total Epoches: {}'.format(
trainer.args.epochs))
args.time_begin = datetime.now()
trainer.validation(epoch=trainer.args.start_epoch)
for epoch in range(trainer.args.start_epoch, trainer.args.epochs):
trainer.training(epoch)
if not trainer.args.no_val and (epoch % args.eval_interval
== (args.eval_interval - 1) or epoch >=
(trainer.args.epochs - 10)):
trainer.validation(epoch)
def __init__(self, options):
super(TSanReportExtractor,
self).__init__(options, Sanitizer('ThreadSanitizer', 'tsan'))
self.__printer = Printer(options)
def __init__(self, options):
self.__options = options
self.__printer = Printer(options)
self.__tasks = []
def setup(self, options):
self.__printer = Printer(options)
# sanitizer_name.category_name.(new|old)
self.__data = OrderedDict()
def setup(self, options):
self.__printer = Printer(options)