def run_main(): args = arguments.get_arguments() pp.pprint(args._to_dicts()) p, conn = None, None env = None random.seed(args.seed) torch.manual_seed(args.seed) choice, input_args = get_choice(args.input) if choice == 'remote': print("connecting to: " + input_args.host) p, conn = connection.connect('ws://' + input_args.host) else: config, dataset = load_dataset(args) env = initialise(config, dataset, args) try: run_trainer(args, conn, env=env) except (KeyboardInterrupt, SystemExit): p.terminate() except Exception: traceback.print_exc() p.terminate()
def main(): args = get_arguments() print(args) np.random.seed(args.seed) dataset, train_img_feature, train_data = get_train_data(args) dataset, test_img_feature, test_data, val_answers = get_data_test(args) train_X = [train_data[u'question'], train_img_feature] train_Y = np_utils.to_categorical(train_data[u'answers'], args.nb_classes) test_X = [test_data[u'question'], test_img_feature] test_Y = np_utils.to_categorical(val_answers, args.nb_classes) model_name = importlib.import_module("models." + args.model) model = model_name.model(args) model.compile(loss='categorical_crossentropy', optimizer=args.optimizer, metrics=['accuracy']) model.summary() # prints model layers with weights history = model.fit(train_X, train_Y, batch_size=args.batch_size, nb_epoch=args.nb_epoch, validation_data=(test_X, test_Y)) return history.history
def test_setup_network(): # NOTE: Removing any sys.argv to get default arguments sys.argv = [""] args = get_arguments() device = "cuda" if torch.cuda.is_available() else "cpu" segmenter, training_loss, validation_loss = setup_network(args, device) assert isinstance(segmenter, torch.nn.Module) assert isinstance(training_loss, torch.nn.CrossEntropyLoss) assert isinstance(validation_loss, dt.engine.MeanIoU)
def test_setup_data_loaders(mocker): # NOTE: Removing any sys.argv to get default arguments sys.argv = [""] args = get_arguments() mocker.patch.object(train, "get_datasets", side_effect=get_fake_datasets) train_loaders, val_loader = train.setup_data_loaders(args) assert len(train_loaders) == args.num_stages for train_loader in train_loaders: assert isinstance(train_loader, torch.utils.data.DataLoader) assert isinstance(val_loader, torch.utils.data.DataLoader)
def code_init(): args = get_arguments() pathdir_init(args) if args.mode == 'train' and args.submode == 'mutual_kd': pass else: print("Using random seed:{}".format(args.seeds)) seed_torch(seed=args.seeds) if args.submode == 'mutual': args.loop = args.mutual_model_num sys.stdout = Logger(osp.join(args.save_dir, 'Log.txt')) printargs(args) return args
def main(): args = get_arguments() logger = logging.getLogger(__name__) torch.backends.cudnn.deterministic = True dt.misc.set_seed(args.random_seed) device = "cuda" if torch.cuda.is_available() else "cpu" # Network segmenter, training_loss, validation_loss = setup_network(args, device=device) # Checkpoint saver, epoch_start = setup_checkpoint_and_maybe_restore(args, model=segmenter) # Data train_loaders, val_loader = setup_data_loaders(args) # Optimisers optimisers, schedulers = setup_optimisers_and_schedulers(args, model=segmenter) total_epoch = epoch_start for stage, num_epochs in enumerate(args.epochs_per_stage): if stage > 0: epoch_start = 0 for epoch in range(epoch_start, num_epochs): logger.info(f"Training: stage {stage} epoch {epoch}") dt.engine.train( model=segmenter, opts=optimisers, crits=training_loss, dataloader=train_loaders[stage], freeze_bn=args.freeze_bn[stage], ) total_epoch += 1 for scheduler in schedulers: scheduler.step(total_epoch) if (epoch + 1) % args.val_every[stage] == 0: logger.info(f"Validation: stage {stage} epoch {epoch}") vals = dt.engine.validate( model=segmenter, metrics=validation_loss, dataloader=val_loader, ) saver.maybe_save( new_val=vals, dict_to_save={ "state_dict": segmenter.state_dict(), "epoch": total_epoch, }, )
def main(): # Read config file for date current_date = get_current_date() # Get command line arguments args = get_arguments() # Check date handling commands and execute the corresponding routine if args.advance_date is not None: current_date = print(advance_date(args.advance_date)) return current_date if args.set_date is not None: current_date = print(set_date(args.set_date)) return current_date # Check commands and execute the corresponding routine if args.CLI_command.lower() == 'buy': print( buy(args.product_name, args.buy_date, args.price, args.expiration_date)) elif args.CLI_command.lower() == 'sell': print(sell(args.product_name, args.sell_date, args.price)) elif args.CLI_command.lower() == 'report': # Convert yesterday, now, today or date to date report_date = None if args.yesterday is not None: report_date = (datetime.strptime(current_date, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y-%m-%d') if args.now is not None: report_date = current_date if args.today is not None: report_date = current_date if args.date is not None: report_date = args.date if report_date is not None: print( report.show_report(args.report_name, report_date, args.export_csv, args.show_graph)) else: print(f"ERROR: missing <date>") # Unknown command else: print( f"ERROR: unknown command '{args.CLI_command}' <buy, sell, report>") return
def main(): output, repo, domain = get_arguments() archive = 'https://{0}/{1}/archive/master.zip'.format(domain, repo) time_format = datetime.now().strftime('%Y-%m-%d_%H.%M.%S') utils.LOG.info('Starting repository analysis...') output = output + '/' + repo.replace('/','_') + '-' + time_format utils.get_package(archive, output) feature_files = utils.get_feature_files(output) tag_ojb_list = utils.parse_files(feature_files) utils.stats_output(tag_ojb_list) utils.LOG.info('Finished...')
def main(): args = get_arguments('generate') try: checkpoint = torch.load("models/" + args.dataset + "/model.pt") except OSError: print("No model exists at models/" + args.dataset + ". You must train the model" + " before generation.") net, _ = get_model_and_data(checkpoint['args']) net.load_state_dict(checkpoint['model']) for _ in range(args.num_samples): print( net.get_sample(init_string=args.init_string, max_length=args.max_length, temperature=args.temperature))
def test_setup_optimisers_and_schedulers(): # NOTE: Removing any sys.argv to get default arguments sys.argv = [""] args = get_arguments() model = DummyEncDecModel() optimisers, schedulers = setup_optimisers_and_schedulers(args, model) assert len(optimisers) == 2 assert len(schedulers) == 2 for optimiser in optimisers: assert isinstance(optimiser, torch.optim.Optimizer) assert hasattr(optimiser, "state_dict") assert hasattr(optimiser, "load_state_dict") assert hasattr(optimiser, "step") assert hasattr(optimiser, "zero_grad") for scheduler in schedulers: assert isinstance(scheduler, torch.optim.lr_scheduler._LRScheduler) assert hasattr(scheduler, "state_dict") assert hasattr(scheduler, "load_state_dict") assert hasattr(scheduler, "step")
import torch.nn as nn import torch.nn.functional as F from datetime import datetime from tqdm import tqdm from pose_dataset import * from pose_resnet import * from pose_resnet_2d import * from pose_hrnet import * import arguments from make_log import * from evaluate import * from loss import * args = arguments.get_arguments() # model name model_name = '{}_nlayer{}_{}_lr{}_batch{}_momentum{}_schedule{}_nepoch{}_{}'.format( args.name, args.nlayer, args.optimizer, args.lr, args.batch_size, args.momentum, args.schedule, args.nepochs, args.arch ) logger = make_logger(log_file=model_name) logger.info("saved model name "+model_name)
def main(): args = get_arguments() logger = logging.getLogger(__name__) torch.backends.cudnn.deterministic = True dt.misc.set_seed(args.random_seed) device = "cuda" if torch.cuda.is_available() else "cpu" # Network segmenter, training_loss, validation_loss = setup_network(args, device=device) # Data train_loaders, val_loader = setup_data_loaders(args) # Optimisers optimisers, schedulers = setup_optimisers_and_schedulers(args, model=segmenter) # Checkpoint saver, restart_epoch = setup_checkpoint_and_maybe_restore( args, model=segmenter, optimisers=optimisers, schedulers=schedulers, ) # Calculate from which stage and which epoch to restart the training total_epoch = restart_epoch all_epochs = np.cumsum(args.epochs_per_stage) restart_stage = sum(restart_epoch >= all_epochs) if restart_stage > 0: restart_epoch -= all_epochs[restart_stage - 1] for stage in range(restart_stage, args.num_stages): if stage > restart_stage: restart_epoch = 0 for epoch in range(restart_epoch, args.epochs_per_stage[stage]): logger.info(f"Training: stage {stage} epoch {epoch}") dt.engine.train( model=segmenter, opts=optimisers, crits=training_loss, dataloader=train_loaders[stage], freeze_bn=args.freeze_bn[stage], grad_norm=args.grad_norm[stage], ) total_epoch += 1 for scheduler in schedulers: scheduler.step(total_epoch) if (epoch + 1) % args.val_every[stage] == 0: logger.info(f"Validation: stage {stage} epoch {epoch}") vals = dt.engine.validate( model=segmenter, metrics=validation_loss, dataloader=val_loader, ) saver.maybe_save( new_val=vals, dict_to_save={ "model": segmenter.state_dict(), "epoch": total_epoch, "optimisers": [optimiser.state_dict() for optimiser in optimisers], "schedulers": [scheduler.state_dict() for scheduler in schedulers], }, )
from bots import * from Player import Player # Change these to edit the default Game parameters DEFAULT_VERBOSITY = True DEFAULT_MIN_ROUNDS = 300 DEFAULT_AVERAGE_ROUNDS = 1000 DEFAULT_END_EARLY = False DEFAULT_PLAYERS = ([Player()] * 5) + ([Player1()] * 5) + ([Player2()] * 5) + ([Player3(0.0001)] * 5) + ([Freeloader()] * 5) + ([Alternator()] * 1) + ([MaxRepHunter()] * 5) + ([Random(0.3)] * 2) + ([Random(0.96)] * 2) + ([FairHunter()] * 2) + ([BoundedHunter(0.8,0.999)] * 5) + ([AverageHunter()] * 3) + ([Pushover()] * 1) # Bare minimum test game. See README.md for details. if __name__ == '__main__': (players, options) = arguments.get_arguments() # The list of players for the game is made up of # 'Player' (your strategy) # bots from get_arguments (the bots to use) player_list = players # **options -> interpret game options from get_arguments # as a dictionary to unpack into the Game parametersi winnerMap = {} for i in range(200) : game = Game(player_list, **options) winner = game.play_game() print ("Winner: " + winner) sys.stdout.flush() if (not winner in winnerMap):
def main(): # Get all the command-line arguments args = arguments.get_arguments() if args.log_file != None: logging.basicConfig(format='%(asctime)s %(message)s', filename=args.log_file, filemode='a', level=logging.DEBUG) else: logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) # Get all the command-line arguments logging.info('Obtained all arguments: %s', str(args)) logging.info('parsing all data') if args.train_data: train_data = parse_file(args.train_data, args.dont_preprocess_data, args.dont_lowercase_words, args.just_pad_sents) if args.dev_data: dev_data = parse_file(args.dev_data, args.dont_preprocess_data, args.dont_lowercase_words, args.just_pad_sents) if args.test_data: test_data = parse_file(args.test_data, args.dont_preprocess_data, args.dont_lowercase_words, args.just_pad_sents) logging.info('train_length: %d', len(train_data)) logging.info('dev_length: %d', len(dev_data)) logging.info('test_length: %d', len(test_data)) char_to_ix, word_to_ix, label_to_ix = get_word_label_ix( train_data, args.vocab_size) if len(word_to_ix) != args.vocab_size: logging.info('vocab_size changed to %d', len(word_to_ix)) args.vocab_size = len(word_to_ix) if args.word_embeds_file != None and args.word_embeds_file != 'None': logging.info('obtaining %s embeddings for words', args.word_embeds) word_vectors = extract_embeds(args.word_embeds_file, args.word_embed_dim, word_to_ix) else: word_vectors = None logging.info('Obtained word_to_ix: %d and label_to_ix: %d ', len(word_to_ix), len(label_to_ix)) logging.info(label_to_ix) if args.model == 'exact_summarunner': logging.info('using a word-level bilstm and sent-level bilstm model') summarunner = model.summarunner(args, word_to_ix, label_to_ix, word_vectors) logging.info('Created the network') logging.info('Training the network') # training the network train(summarunner, train_data, dev_data, test_data, args) else: logging.error('no such option for the model yet')
from arguments import get_arguments from combine import Combine from models.vgg import * from models.dpn import * from models.lenet import * from models.senet import * from models.resnet import * from models.resnext import * from models.densenet import * from models.googlenet import * from models.mobilenet import * from models.shufflenet import * from models.preact_resnet import * from models.smallnet import * parser = get_arguments() opt = parser.parse_args() img_size = (1, 3, 32, 32) inputs = torch.randn(1, 3, 32, 32) # resnet18 = models.resnet18() # model = VGG16() arch_config = {} arch_config['num_classes'] = 10 arch_config['num_channels'] = 3 if opt.E > 1: arch_config['balance'] = opt.balance else: arch_config['balance'] = False model = Combine(opt.arch, opt.E, opt.probs, ensemble=True, **arch_config) # model = SmallNetOriginal() y, _ = model(Variable(inputs))
def main(): """ Create main function to capture code errors: https://stackoverflow.com/questions/6234405/logging-uncaught-exceptions-in-python """ args = get_arguments() ###################################################################### #####################START PIPELINE################################### ###################################################################### output = os.path.abspath(args.output) group_name = output.split("/")[-1] reference = os.path.abspath(args.reference) #annotation = os.path.abspath(args.annotation) # LOGGING # Create log file with date and time right_now = str(datetime.datetime.now()) right_now_full = "_".join(right_now.split(" ")) log_filename = group_name + "_" + right_now_full + ".log" log_folder = os.path.join(output, 'Logs') check_create_dir(log_folder) log_full_path = os.path.join(log_folder, log_filename) logger = logging.getLogger() logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s:%(message)s') file_handler = logging.FileHandler(log_full_path) file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(formatter) stream_handler = logging.StreamHandler() stream_handler.setLevel(logging.INFO) # stream_handler.setFormatter(formatter) logger.addHandler(stream_handler) logger.addHandler(file_handler) logger.info("\n\n" + BLUE + BOLD + "STARTING PIPELINE IN GROUP: " + group_name + END_FORMATTING) today = str(datetime.date.today()) logger.info("ARGUMENTS:") logger.info(str(args)) # Obtain all R1 and R2 from folder r1, r2 = extract_read_list(args.input_dir) # Check if there are samples to filter out sample_list_F = [] if args.sample_list == None: logger.info("\n" + "No samples to filter") for r1_file, r2_file in zip(r1, r2): sample = extract_sample(r1_file, r2_file) sample_list_F.append(sample) else: logger.info("samples will be filtered") sample_list_F = file_to_list(args.sample_list) new_samples = check_reanalysis(args.output, sample_list_F) logger.info("\n%d samples will be analysed: %s" % (len(sample_list_F), ",".join(sample_list_F))) logger.info("\n%d NEW samples will be analysed: %s" % (len(new_samples), ",".join(new_samples))) #DECLARE FOLDERS CREATED IN PIPELINE ################ #AND KEY FILES ###################################### ##################################################### # Annotation related parameters #script_dir = os.path.dirname(os.path.realpath(__file__)) # Output related out_qc_dir = os.path.join(output, "Quality") out_qc_pre_dir = os.path.join(out_qc_dir, "raw") # subfolder out_variant_dir = os.path.join(output, "Variants") out_core_dir = os.path.join(output, "Core") out_stats_dir = os.path.join(output, "Stats") out_stats_bamstats_dir = os.path.join( out_stats_dir, "Bamstats") # subfolder out_stats_coverage_dir = os.path.join( out_stats_dir, "Coverage") # subfolder out_compare_dir = os.path.join(output, "Compare") out_annot_dir = os.path.join(output, "Annotation") out_annot_snpeff_dir = os.path.join(out_annot_dir, "snpeff") # subfolder out_annot_user_dir = os.path.join(out_annot_dir, "user") # subfolder out_annot_user_aa_dir = os.path.join(out_annot_dir, "user_aa") # subfolder out_annot_blast_dir = os.path.join(out_annot_dir, "blast") # subfolder out_species_dir = os.path.join(output, "Species") new_sample_number = 0 for r1_file, r2_file in zip(r1, r2): # EXtract sample name sample = extract_sample(r1_file, r2_file) args.sample = sample if sample in sample_list_F: # VARINAT SAMPLE DIR sample_variant_dir = os.path.join(out_variant_dir, sample) sample_number = str(sample_list_F.index(sample) + 1) sample_total = str(len(sample_list_F)) if sample in new_samples: new_sample_number = str(int(new_sample_number) + 1) new_sample_total = str(len(new_samples)) logger.info("\n" + WHITE_BG + "STARTING SAMPLE: " + sample + " (" + sample_number + "/" + sample_total + ")" + " (" + new_sample_number + "/" + new_sample_total + ")" + END_FORMATTING) else: logger.info("\n" + WHITE_BG + "STARTING SAMPLE: " + sample + " (" + sample_number + "/" + sample_total + ")" + END_FORMATTING) output_final_vcf = os.path.join( sample_variant_dir, 'snps.all.ivar.tsv') if not os.path.isfile(output_final_vcf): ##############START PIPELINE##################### ################################################# # INPUT ARGUMENTS ################ # check_file_exists(r1_file) # check_file_exists(r2_file) args.output = os.path.abspath(args.output) check_create_dir(args.output) # QUALITY CHECK in RAW with fastqc ###################################################### check_create_dir(out_qc_dir) out_qc_raw_name_r1 = (".").join(r1_file.split( '/')[-1].split('.')[0:-2]) + '_fastqc.html' out_qc_raw_name_r2 = (".").join(r2_file.split( '/')[-1].split('.')[0:-2]) + '_fastqc.html' output_qc_raw_file_r1 = os.path.join( out_qc_pre_dir, out_qc_raw_name_r1) output_qc_raw_file_r2 = os.path.join( out_qc_pre_dir, out_qc_raw_name_r2) if os.path.isfile(output_qc_raw_file_r1) and os.path.isfile(output_qc_raw_file_r2): logger.info(YELLOW + DIM + output_qc_raw_file_r1 + " EXIST\nOmmiting QC for sample " + sample + END_FORMATTING) else: logger.info( GREEN + "Checking quality in sample " + sample + END_FORMATTING) logger.info("R1: " + r1_file + "\nR2: " + r2_file) fastqc_quality(r1_file, r2_file, out_qc_pre_dir, args.threads) """ TODO: Human filter """ # VARIANT CALLING WITH SNIPPY ################################################### output_vcf_sub = os.path.join( sample_variant_dir, "snps.subs.vcf") output_vcf = os.path.join(sample_variant_dir, "snps.vcf") if os.path.isfile(output_vcf_sub) and os.path.isfile(output_vcf): logger.info(YELLOW + DIM + output_vcf + " EXIST\nOmmiting Variant calling in " + sample + END_FORMATTING) else: logger.info( GREEN + "Calling variants with snippy " + sample + END_FORMATTING) run_snippy(r1_file, r2_file, reference, out_variant_dir, sample, threads=args.threads, minqual=10, minfrac=0.1, mincov=1) old_bam = os.path.join(sample_variant_dir, "snps.bam") old_bai = os.path.join(sample_variant_dir, "snps.bam.bai") new_bam = os.path.join(sample_variant_dir, sample + ".bam") new_bai = os.path.join( sample_variant_dir, sample + ".bam.bai") os.rename(old_bam, new_bam) os.rename(old_bai, new_bai) #VARIANT FORMAT COMBINATION (REMOVE COMPLEX) ######## ##################################################### out_variant_indel_sample = os.path.join( sample_variant_dir, "snps.indel.vcf") out_variant_all_sample = os.path.join( sample_variant_dir, "snps.all.vcf") if os.path.isfile(out_variant_indel_sample): logger.info(YELLOW + DIM + out_variant_indel_sample + " EXIST\nOmmiting indel filtering in sample " + sample + END_FORMATTING) else: logger.info(GREEN + "Filtering INDELS in " + sample + END_FORMATTING) extract_indels(output_vcf) if os.path.isfile(out_variant_all_sample): logger.info(YELLOW + DIM + out_variant_all_sample + " EXIST\nOmmiting vcf combination in sample " + sample + END_FORMATTING) else: logger.info(GREEN + "Combining vcf in " + sample + END_FORMATTING) merge_vcf(output_vcf_sub, out_variant_indel_sample) #VARIANT FORMAT ADAPTATION TO IVAR ################## ##################################################### out_variant_tsv_file = os.path.join( sample_variant_dir, 'snps.all.ivar.tsv') if os.path.isfile(out_variant_tsv_file): logger.info(YELLOW + DIM + out_variant_tsv_file + " EXIST\nOmmiting format adaptation for sample " + sample + END_FORMATTING) else: logger.info( GREEN + "Adapting variants format in sample " + sample + END_FORMATTING) prior = datetime.datetime.now() vcf_to_ivar_tsv(out_variant_all_sample, out_variant_tsv_file) after = datetime.datetime.now() print(("Done with function in: %s" % (after - prior))) # SPECIES DETERMINATION ################################################### check_create_dir(out_species_dir) output_species = os.path.join( out_species_dir, sample + ".screen.tab") if os.path.isfile(output_species): logger.info(YELLOW + DIM + output_species + " EXIST\nOmmiting Species determinatin in " + sample + END_FORMATTING) else: logger.info( GREEN + "Determining species in " + sample + END_FORMATTING) mash_screen(r1_file, out_species_dir, r2_file=r2_file, winner=True, threads=args.threads, mash_database=args.mash_database) ########################CREATE STATS AND QUALITY FILTERS######################################################################## ################################################################################################################################ #CREATE Bamstats####################################### ####################################################### check_create_dir(out_stats_dir) check_create_dir(out_stats_bamstats_dir) out_bamstats_name = sample + ".bamstats" out_bamstats_file = os.path.join( out_stats_bamstats_dir, out_bamstats_name) bam_sample_file = os.path.join(sample_variant_dir, sample + ".bam") if os.path.isfile(out_bamstats_file): logger.info(YELLOW + DIM + out_bamstats_file + " EXIST\nOmmiting Bamstats for sample " + sample + END_FORMATTING) else: logger.info(GREEN + "Creating bamstats in sample " + sample + END_FORMATTING) create_bamstat( bam_sample_file, out_stats_bamstats_dir, sample, threads=args.threads) #CREATE Bamstats####################################### ####################################################### check_create_dir(out_stats_coverage_dir) out_coverage_name = sample + ".cov" out_coverage_file = os.path.join( out_stats_coverage_dir, out_coverage_name) if os.path.isfile(out_coverage_file): logger.info(YELLOW + DIM + out_coverage_file + " EXIST\nOmmiting Bamstats for sample " + sample + END_FORMATTING) else: logger.info(GREEN + "Creating coverage in sample " + sample + END_FORMATTING) create_coverage(bam_sample_file, out_stats_coverage_dir, sample) # coverage OUTPUT SUMMARY ###################################################### prior_recal = datetime.datetime.now() logger.info(GREEN + "Creating summary report for coverage result in group " + group_name + END_FORMATTING) obtain_group_cov_stats(out_stats_dir, group_name) after_recal = datetime.datetime.now() logger.info("Done with report for coverage: %s" % (after_recal - prior_recal)) # READS and VARIANTS OUTPUT SUMMARY ###################################################### logger.info(GREEN + "Creating overal summary report in group " + group_name + END_FORMATTING) obtain_overal_stats(output, group_name) # REMOVE UNCOVERED ############################################################################################################################## logger.info(GREEN + "Removing low quality samples in group " + group_name + END_FORMATTING) uncovered_samples = remove_low_quality( output, min_coverage=args.coverage20, min_hq_snp=args.min_snp, type_remove='Uncovered') if len(uncovered_samples) > 1: logger.info(GREEN + "Uncovered samples: " + (",").join(uncovered_samples) + END_FORMATTING) else: logger.info(GREEN + "NO uncovered samples found" + END_FORMATTING) # RUN SNIPPY CORE ############################################################################################################################## if args.core: check_create_dir(out_core_dir) logger.info(GREEN + "Running snippy-core " + group_name + END_FORMATTING) run_snippy_core(out_variant_dir, out_core_dir, reference) logger.info(GREEN + "Adapting core-snp to compare format " + group_name + END_FORMATTING) core_vcf_file = os.path.join(out_core_dir, "core.vcf") core_vcf_file_adapted = os.path.join( out_core_dir, "core.vcf.adapted.tsv") core_vcf_file_removed = os.path.join( out_core_dir, "core.vcf.adapted.final.tsv") core_vcf_df_adapted = import_VCF4_core_to_compare(core_vcf_file) core_vcf_df_adapted.to_csv( core_vcf_file_adapted, sep="\t", index=False) logger.info(GREEN + "Obtaining clustered positions " + group_name + END_FORMATTING) close_positions_list = extract_close_snps( core_vcf_df_adapted, snps_in_10=1) logger.info(GREEN + "Obtaining uncovered positions " + group_name + END_FORMATTING) uncovered_list = identify_uncovered( out_stats_coverage_dir, min_coverage=10, nocall_fr=0.5) logger.debug('Clustered positions in core SNP:\n{}'.format( (",".join([str(x) for x in close_positions_list])))) logger.debug('Uncovered positions in all samples:\n{}'.format( (",".join([str(x) for x in uncovered_list])))) to_remove_list = close_positions_list + uncovered_list remove_df = remove_position_from_compare( core_vcf_df_adapted, to_remove_list) remove_df.to_csv(core_vcf_file_removed, sep="\t", index=False) ddtb_compare(core_vcf_file_removed, distance=10) #ANNOTATION WITH SNPEFF AND USER INPUT ############## ##################################################### logger.info("\n\n" + BLUE + BOLD + "STARTING ANNOTATION IN GROUP: " + group_name + END_FORMATTING + "\n") check_create_dir(out_annot_dir) check_create_dir(out_annot_snpeff_dir) # SNPEFF if args.snpeff_database != False: for root, _, files in os.walk(out_variant_dir): for name in files: if name == 'snps.all.vcf': sample = root.split('/')[-1] filename = os.path.join(root, name) chrom_filename = os.path.join( root, 'snps.all.chromosome.vcf') out_annot_file = os.path.join( out_annot_snpeff_dir, sample + ".annot") if os.path.isfile(out_annot_file): logger.info(YELLOW + DIM + out_annot_file + " EXIST\nOmmiting snpEff Annotation for sample " + sample + END_FORMATTING) else: logger.info( GREEN + "Annotating sample with snpEff: " + sample + END_FORMATTING) rename_reference_snpeff(filename, chrom_filename) annotate_snpeff(chrom_filename, out_annot_file, database=args.snpeff_database) else: logger.info(YELLOW + DIM + " No SnpEff database suplied, skipping annotation in group " + group_name + END_FORMATTING) # USER DEFINED if not args.annot_bed and not args.annot_vcf: logger.info( YELLOW + BOLD + "Ommiting User Annotation, no BED or VCF files supplied" + END_FORMATTING) else: check_create_dir(out_annot_user_dir) for root, _, files in os.walk(out_variant_dir): for name in files: if name == 'snps.all.ivar.tsv': sample = root.split('/')[-1] logger.info( 'User bed/vcf annotation in sample {}'.format(sample)) filename = os.path.join(root, name) out_annot_file = os.path.join( out_annot_user_dir, sample + ".tsv") user_annotation( filename, out_annot_file, vcf_files=args.annot_vcf, bed_files=args.annot_bed) # USER AA DEFINED if not args.annot_aa: logger.info( YELLOW + BOLD + "Ommiting User aa Annotation, no AA files supplied" + END_FORMATTING) else: check_create_dir(out_annot_user_aa_dir) for root, _, files in os.walk(out_annot_snpeff_dir): if root == out_annot_snpeff_dir: for name in files: if name.endswith('.annot'): sample = name.split('.')[0] logger.info( 'User aa annotation in sample {}'.format(sample)) filename = os.path.join(root, name) out_annot_aa_file = os.path.join( out_annot_user_aa_dir, sample + ".tsv") if os.path.isfile(out_annot_aa_file): user_annotation_aa( out_annot_aa_file, out_annot_aa_file, aa_files=args.annot_aa) else: user_annotation_aa( filename, out_annot_aa_file, aa_files=args.annot_aa) # USER FASTA ANNOTATION if not args.annot_fasta: logger.info( YELLOW + BOLD + "Ommiting User FASTA Annotation, no FASTA files supplied" + END_FORMATTING) else: check_create_dir(out_annot_blast_dir) for root, _, files in os.walk(out_variant_dir): for name in files: if name.endswith('.consensus.subs.fa'): filename = os.path.join(root, name) sample = root.split('/')[-1] logger.info( 'User FASTA annotation in sample {}'.format(sample)) # out_annot_aa_file = os.path.join( # out_annot_user_aa_dir, sample + ".tsv") for db in args.annot_fasta: make_blast(filename, db, sample, out_annot_blast_dir, db_type="nucl", query_type="nucl", evalue=0.0001, threads=8) # USER AA TO HTML if not args.annot_aa: logger.info( YELLOW + BOLD + "Ommiting User aa Annotation to HTML, no AA files supplied" + END_FORMATTING) else: annotated_samples = [] logger.info('Adapting annotation to html in {}'.format(group_name)) for root, _, files in os.walk(out_annot_user_aa_dir): if root == out_annot_user_aa_dir: for name in files: if name.endswith('.tsv'): sample = name.split('.')[0] annotated_samples.append(sample) filename = os.path.join(root, name) annotation_to_html(filename, sample) annotated_samples = [str(x) for x in annotated_samples] report_samples_html_all = report_samples_html.replace( 'ALLSAMPLES', ('","').join(annotated_samples)) # NEW with open(os.path.join(out_annot_user_aa_dir, '00_all_samples.html'), 'w+') as f: f.write(report_samples_html_all) # SNP COMPARISON using tsv variant files ###################################################### logger.info("\n\n" + BLUE + BOLD + "STARTING COMPARISON IN GROUP: " + group_name + END_FORMATTING + "\n") check_create_dir(out_compare_dir) folder_compare = today + "_" + group_name path_compare = os.path.join(out_compare_dir, folder_compare) check_create_dir(path_compare) full_path_compare = os.path.join(path_compare, group_name) compare_snp_matrix_recal = full_path_compare + ".revised.final.tsv" compare_snp_matrix_recal_intermediate = full_path_compare + ".revised_intermediate.tsv" compare_snp_matrix_recal_mpileup = full_path_compare + \ ".revised_intermediate_vcf.tsv" compare_snp_matrix_INDEL_intermediate = full_path_compare + \ ".revised_INDEL_intermediate.tsv" # Create intermediate recalibrated_snp_matrix_intermediate = ddbb_create_intermediate( out_variant_dir, out_stats_coverage_dir, min_freq_discard=0.1, min_alt_dp=10, only_snp=False) # recalibrated_snp_matrix_intermediate.to_csv( # compare_snp_matrix_recal_intermediate, sep="\t", index=False) # Remove SNPs from BED file (PE/PPE) if args.remove_bed: recalibrated_snp_matrix_intermediate = remove_bed_positions( recalibrated_snp_matrix_intermediate, args.remove_bed) recalibrated_snp_matrix_intermediate.to_csv( compare_snp_matrix_recal_intermediate, sep="\t", index=False) # Recalibrate intermediate with VCF prior_recal = datetime.datetime.now() recalibrated_snp_matrix_mpileup = recalibrate_ddbb_vcf_intermediate( compare_snp_matrix_recal_intermediate, out_variant_dir, min_cov_low_freq=10) recalibrated_snp_matrix_mpileup.to_csv( compare_snp_matrix_recal_mpileup, sep="\t", index=False) after_recal = datetime.datetime.now() logger.debug("Done with recalibration vcf: %s" % (after_recal - prior_recal)) # Remove SNPs located within INDELs compare_snp_matrix_INDEL_intermediate_df = remove_position_range( recalibrated_snp_matrix_mpileup) compare_snp_matrix_INDEL_intermediate_df.to_csv( compare_snp_matrix_INDEL_intermediate, sep="\t", index=False) # Extract all positions marked as complex complex_variants = extract_complex_list(out_variant_dir) logger.debug('Complex positions in all samples:\n{}'.format( (",".join([str(x) for x in complex_variants])))) # Clean all faulty positions and samples => Final table recalibrated_revised_INDEL_df = revised_df(compare_snp_matrix_INDEL_intermediate_df, path_compare, complex_pos=complex_variants, min_freq_include=0.8, min_threshold_discard_uncov_sample=args.min_threshold_discard_uncov_sample, min_threshold_discard_uncov_pos=args.min_threshold_discard_uncov_pos, min_threshold_discard_htz_sample=args.min_threshold_discard_htz_sample, min_threshold_discard_htz_pos=args.min_threshold_discard_htz_pos, min_threshold_discard_all_pos=args.min_threshold_discard_all_pos, min_threshold_discard_all_sample=args.min_threshold_discard_all_sample, remove_faulty=True, drop_samples=True, drop_positions=True, windows_size_discard=args.window) recalibrated_revised_INDEL_df.to_csv( compare_snp_matrix_recal, sep="\t", index=False) # Matrix to pairwise and mwk ddtb_compare(compare_snp_matrix_recal, distance=5) logger.info("\n\n" + MAGENTA + BOLD + "COMPARING FINISHED IN GROUP: " + group_name + END_FORMATTING + "\n") logger.info("\n\n" + MAGENTA + BOLD + "#####END OF PIPELINE AUTOSNIPPY ANALYSIS#####" + END_FORMATTING + "\n")
from Player import Player # Change these to edit the default Game parameters DEFAULT_VERBOSITY = False DEFAULT_QUIET = False DEFAULT_MIN_ROUNDS = 300 DEFAULT_AVERAGE_ROUNDS = 1000 DEFAULT_END_EARLY = False DEFAULT_PLAYERS = [ Player(), Pushover(), Freeloader(), Alternator(), MaxRepHunter(), Random(.2), Random(.8) ] # Bare minimum test game. See README.md for details. if __name__ == '__main__': (players, options) = arguments.get_arguments() # The list of players for the game is made up of # 'Player' (your strategy) # bots from get_arguments (the bots to use) player_list = players # **options -> interpret game options from get_arguments # as a dictionary to unpack into the Game parameters game = Game(player_list, **options) game.play_game()
def main(): args = get_arguments('parser') parse_corpus(args.dataset)
break await asyncio.sleep(1) if ('error' in response): raise ValueError('Timeout exception[' + str(i) + '].') logger.log('Polling[' + str(i) + '] complete!') response = response['response']['result'] data = json.loads(response['value']) responses[i] = data if __name__ == "__main__": """ Gets the arguments and makes the requests according to them. """ args = get_arguments() responses = [None] * args.requests loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather( *[ga_action_request(args, i) for i in range(0, args.requests)] )) loop.close() if (args.only_population): population = [ individual for response in responses for individual in response['population'] ] print(population) else: print(responses)
#!/usr/bin/python3 import re from glob import glob from copy import deepcopy from math import log from arguments import get_arguments DOC_DIR = 'test_docs' NOT_WORD_RE = '[^\w]' ops = {'and': set.intersection, 'or': set.union, 'not': set.difference} args = get_arguments() def idf(term, docs): N = len(docs) df = count_docs_containing_term(term, [doc[1] for doc in docs]) if df == 0: return 0 else: return log(N/df) def tf(term, doc): count = 0 for word in doc: if word == term: count += 1 return count
from __future__ import division, print_function import arguments from Game import Game from bots import * from Player import Player # Change these to edit the default Game parameters DEFAULT_VERBOSITY = True DEFAULT_MIN_ROUNDS = 300 DEFAULT_AVERAGE_ROUNDS = 1000 DEFAULT_END_EARLY = False # Bare minimum test game. See README.md for details. if __name__ == '__main__': (bots, options) = arguments.get_arguments() # The list of players for the game is made up of # 'Player' (your strategy) # bots from get_arguments (the bots to use) players = [Player()] + bots # **options -> interpret game options from get_arguments # as a dictionary to unpack into the Game parameters game = Game(players, **options) game.play_game()
def main(): args = arguments.get_arguments() if args.log_file != None: logging.basicConfig(format='%(asctime)s %(message)s', filename=args.log_file, filemode='a', level=logging.DEBUG) else: logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) #logging.basicConfig(format='%(asctime)s %(message)s', filename='example.log', filemode='w', level=logging.DEBUG) #logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) #logging.debug('This message should go to the log file') #logging.info('So should this') #logging.warning('And this, too') # Get all the command-line arguments #args = arguments.get_arguments() logging.info('Obtained all arguments: %s', str(args)) if args.train_data: train_data = parse_file(args.train_data, args.dont_preprocess_data, args.dont_lowercase_words, args.share_hiddenstates, args.just_pad_sents) if args.dev_data: dev_data = parse_file(args.dev_data, args.dont_preprocess_data, args.dont_lowercase_words, args.share_hiddenstates, args.just_pad_sents) if args.test_data: test_data = parse_file(args.test_data, args.dont_preprocess_data, args.dont_lowercase_words, args.share_hiddenstates, args.just_pad_sents) if args.test_data2: test_data2 = parse_file(args.test_data2, args.dont_preprocess_data, args.dont_lowercase_words, args.share_hiddenstates, args.just_pad_sents) if args.model == 'char_bilstm' or args.model == 'char_bilstm-crf': if args.train_data: #train_data_word = train_data train_data_word = parse_file(args.train_data, True, True, args.share_hiddenstates, args.just_pad_sents) #train_data_word = parse_file2(args.train_data, args.dont_lowercase_words, args.share_hiddenstates) if args.dev_data: #dev_data_word = dev_data dev_data_word = parse_file(args.dev_data, True, True, args.share_hiddenstates, args.just_pad_sents) #dev_data_word = parse_file2(args.dev_data, args.dont_lowercase_words, args.share_hiddenstates) if args.test_data: #test_data_word = test_data test_data_word = parse_file(args.test_data, True, True, args.share_hiddenstates, args.just_pad_sents) #test_data_word = parse_file2(args.test_data, args.dont_lowercase_words, args.share_hiddenstates) if args.test_data2: #test_data2_word = test_data2 test_data2_word = parse_file(args.test_data2, True, True, args.share_hiddenstates, args.just_pad_sents) #test_data2_word = parse_file2(args.test_data2, args.dont_lowercase_words, args.share_hiddenstates) if args.combine_train_dev: logging.info('combining training and dev data') train_data = train_data + dev_data if args.model == 'char_bilstm' or args.model == 'char_bilstm-crf': train_data_word = train_data_word + dev_data_word if args.combine_train_dev_test: logging.info('combining training, dev data and test data') train_data = train_data + dev_data + test_data if args.model == 'char_bilstm' or args.model == 'char_bilstm-crf': train_data_word = train_data_word + dev_data_word + test_data_word #Check the initializer if needed # Declare a DynetParams object #dyparams = dy.DynetParams() # Fetch the command line arguments (optional) #dyparams.from_args() # Set some parameters manualy (see the command line arguments documentation) #dyparams.set_mem(2048) #dyparams.set_random_seed(666) # Initialize with the given parameters #dyparams.init() # or init_from_params(dyparams) logging.info('parsing all data') logging.info('train_length: %d', len(train_data)) logging.info('dev_length: %d', len(dev_data)) logging.info('test_length: %d', len(test_data)) logging.info('test_length2: %d', len(test_data2)) _, word_to_ix, tag_to_ix = get_word_label_ix(train_data, args.vocab_size) if args.model == 'char_bilstm' or args.model == 'char_bilstm-crf': char_to_ix, _, _ = get_word_label_ix(train_data_word, args.vocab_size) if len(word_to_ix) != args.vocab_size: logging.info('vocab_size changed to %d', len(word_to_ix)) args.vocab_size = len(word_to_ix) if args.word_embeds_file != None and args.word_embeds_file != 'None': logging.info('obtaining %s embeddings for words', args.word_embeds) word_vectors = extract_embeds(args.word_embeds_file, args.word_embed_dim, word_to_ix) ''' if args.use_pretrained_embed2: word_vectors2 = extract_embeds(args.word_embeds_file2, args.word_embed_dim, word_to_ix) new_word_vectors = [] for i,j in zip(word_vectors, word_vectors2): new_word_vectors.append(i+j) word_vectors = new_word_vectors ''' #word_vectors2 = extract_embeds('utils/final_polyglot_embeds.txt', args.word_embed_dim, word_to_ix2) #new_word_vectors = [] #for i,j in zip(word_vectors,word_vectors2): # temp_vector = i+j # new_word_vectors.append(temp_vector) if args.model == 'char_bilstm' or args.model == 'char_bilstm-crf': #logging.info('obtaining %s embeddings for chars',args.word_embeds) #char_vectors = extract_embeds(args.word_embeds_file, args.word_embed_dim, char_to_ix) char_vectors = None else: word_vectors = None if args.model == 'char_bilstm' or args.model == 'char_bilstm-crf': char_vectors = None #word_vectors = extract_embeds('../glove.twitter.27B/glove.twitter.27B.25d.txt', 25, word_to_ix) #word_vectors = None logging.info('Obtained word_to_ix: %d and tag_to_ix: %d ', len(word_to_ix), len(tag_to_ix)) logging.info(tag_to_ix) if args.model == 'char_bilstm': logging.info(char_to_ix) logging.info('using a char-level bilstm and word-level bilstm model') #char_vectors=None char_bilstm = model.ensemble_char_BiLSTMNetwork( args, char_to_ix, word_to_ix, tag_to_ix, args.num_basis, word_vectors, char_vectors) #, new_word_vectors) logging.info('Created the network') logging.info('Training the network') # training the network char_train(char_bilstm, train_data, dev_data, test_data, test_data2, train_data_word, dev_data_word, test_data_word, test_data2_word, args.epochs, args.batch_size, args, tag_to_ix) else: logging.error('no such option for the model yet')
def main(): args = get_arguments('train') net, poems_split_by_words = get_model_and_data(args) random.shuffle(poems_split_by_words) training_num_poems = int( len(poems_split_by_words) * args.training_testing_split) testing_num_poems = len(poems_split_by_words) - training_num_poems training_poems = poems_split_by_words[:training_num_poems] testing_poems = poems_split_by_words[training_num_poems:] training_poems = training_poems[:5] testing_poems = testing_poems[:1] testing_poems = training_poems print("*** Num training poems: {0} ***".format(training_num_poems)) print("*** Num testing poems: {0} ***".format(testing_num_poems)) print("*** Vocabulary size: {0} ***".format(len(net.word2i))) num_params = 0 for param in list(net.parameters()): product_of_dimensions = 1 for dimension in param.size(): product_of_dimensions *= dimension num_params += product_of_dimensions print("*** Num Params: {0} ***".format(num_params)) if not args.no_cuda and torch.cuda.is_available(): print("Created net. Sending to GPU ...") net.cuda() print("GatedLSTM:", net) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=args.lr) split_training_poems = [net.split_poem(poem) for poem in training_poems] split_testing_poems = [net.split_poem(poem) for poem in testing_poems] for epoch in range(args.epochs): for i, (input, output) in enumerate(split_training_poems): loss = train(net, criterion, optimizer, input, output) if i % args.loss_every == args.loss_every - 1: print("Step {0}, Loss {1}".format( epoch * len(split_training_poems) + i, loss)) if i % args.example_every == args.example_every - 1: print("Example:") print(repr(net.get_sample())) print("\n") if i % args.eval_every == args.eval_every - 1: testing_losses = [ train(net, criterion, optimizer, split_poem[0], split_poem[1], update_model=False) for split_poem in split_testing_poems ] testing_loss = sum(testing_losses) / len(testing_losses) print("Validation Loss {0}".format(testing_loss)) checkpoint = { 'model': net.state_dict(), 'optimizer': optimizer.state_dict(), 'args': args } if not os.path.exists("models/" + args.dataset): os.makedirs("models/" + args.dataset) torch.save(checkpoint, "models/" + args.dataset + "/model.pt") print("*** Saved model. ***")
def main(): """Create the model and start the training.""" args = get_arguments() cudnn.enabled = True n_discriminators = 5 # create teacher & student student_net = UNet(3, n_classes=args.num_classes) teacher_net = UNet(3, n_classes=args.num_classes) student_params = list(student_net.parameters()) # teacher doesn't need gradient as it's just a EMA of the student teacher_params = list(teacher_net.parameters()) for param in teacher_params: param.requires_grad = False student_net.train() student_net.cuda(args.gpu) teacher_net.train() teacher_net.cuda(args.gpu) cudnn.benchmark = True unsup_weights = [ args.unsup_weight5, args.unsup_weight6, args.unsup_weight7, args.unsup_weight8, args.unsup_weight9 ] lambda_adv_tgts = [ args.lambda_adv_tgt5, args.lambda_adv_tgt6, args.lambda_adv_tgt7, args.lambda_adv_tgt8, args.lambda_adv_tgt9 ] # create a list of discriminators discriminators = [] for dis_idx in range(n_discriminators): discriminators.append(FCDiscriminator(num_classes=args.num_classes)) discriminators[dis_idx].train() discriminators[dis_idx].cuda(args.gpu) if not os.path.exists(args.snapshot_dir): os.makedirs(args.snapshot_dir) max_iters = args.num_steps * args.iter_size * args.batch_size src_set = REFUGE(True, domain='REFUGE_SRC', is_transform=True, augmentations=aug_student, aug_for_target=aug_teacher, max_iters=max_iters) src_loader = data.DataLoader(src_set, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) src_loader_iter = enumerate(src_loader) tgt_set = REFUGE(True, domain='REFUGE_DST', is_transform=True, augmentations=aug_student, aug_for_target=aug_teacher, max_iters=max_iters) tgt_loader = data.DataLoader(tgt_set, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) tgt_loader_iter = enumerate(tgt_loader) student_optimizer = optim.SGD(student_params, lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) teacher_optimizer = optim_weight_ema.WeightEMA(teacher_params, student_params, alpha=args.teacher_alpha) d_optimizers = [] for idx in range(n_discriminators): optimizer = optim.Adam(discriminators[idx].parameters(), lr=args.learning_rate_D, betas=(0.9, 0.99)) d_optimizers.append(optimizer) calc_bce_loss = torch.nn.BCEWithLogitsLoss() # labels for adversarial training source_label, tgt_label = 0, 1 for i_iter in range(args.num_steps): total_seg_loss = 0 seg_loss_vals = [0] * n_discriminators adv_tgt_loss_vals = [0] * n_discriminators d_loss_vals = [0] * n_discriminators unsup_loss_vals = [0] * n_discriminators for d_optimizer in d_optimizers: d_optimizer.zero_grad() adjust_learning_rate_D(d_optimizer, i_iter, args) student_optimizer.zero_grad() adjust_learning_rate(student_optimizer, i_iter, args) for sub_i in range(args.iter_size): # ******** Optimize source network with segmentation loss ******** # As we don't change the discriminators, their parameters are fixed for discriminator in discriminators: for param in discriminator.parameters(): param.requires_grad = False _, src_batch = src_loader_iter.__next__() _, _, src_images, src_labels, _ = src_batch src_images = Variable(src_images).cuda(args.gpu) # calculate the segmentation losses sup_preds = list(student_net(src_images)) seg_losses, total_seg_loss = [], 0 for idx, sup_pred in enumerate(sup_preds): sup_interp_pred = (sup_pred) # you also can use dice loss like: dice_loss(src_labels, sup_interp_pred) seg_loss = Weighted_Jaccard_loss(src_labels, sup_interp_pred, args.class_weights, args.gpu) seg_losses.append(seg_loss) total_seg_loss += seg_loss * unsup_weights[idx] seg_loss_vals[idx] += seg_loss.item() / args.iter_size _, tgt_batch = tgt_loader_iter.__next__() tgt_images0, tgt_lbl0, tgt_images1, tgt_lbl1, _ = tgt_batch tgt_images0 = Variable(tgt_images0).cuda(args.gpu) tgt_images1 = Variable(tgt_images1).cuda(args.gpu) # calculate ensemble losses stu_unsup_preds = list(student_net(tgt_images1)) tea_unsup_preds = teacher_net(tgt_images0) total_mse_loss = 0 for idx in range(n_discriminators): stu_unsup_probs = F.softmax(stu_unsup_preds[idx], dim=-1) tea_unsup_probs = F.softmax(tea_unsup_preds[idx], dim=-1) unsup_loss = calc_mse_loss(stu_unsup_probs, tea_unsup_probs, args.batch_size) unsup_loss_vals[idx] += unsup_loss.item() / args.iter_size total_mse_loss += unsup_loss * unsup_weights[idx] total_mse_loss = total_mse_loss / args.iter_size # As the requires_grad is set to False in the discriminator, the # gradients are only accumulated in the generator, the target # student network is optimized to make the outputs of target domain # images close to the outputs of source domain images stu_unsup_preds = list(student_net(tgt_images0)) d_outs, total_adv_loss = [], 0 for idx in range(n_discriminators): stu_unsup_interp_pred = (stu_unsup_preds[idx]) d_outs.append(discriminators[idx](stu_unsup_interp_pred)) label_size = d_outs[idx].data.size() labels = torch.FloatTensor(label_size).fill_(source_label) labels = Variable(labels).cuda(args.gpu) adv_tgt_loss = calc_bce_loss(d_outs[idx], labels) total_adv_loss += lambda_adv_tgts[idx] * adv_tgt_loss adv_tgt_loss_vals[idx] += adv_tgt_loss.item() / args.iter_size total_adv_loss = total_adv_loss / args.iter_size # requires_grad is set to True in the discriminator, we only # accumulate gradients in the discriminators, the discriminators are # optimized to make true predictions d_losses = [] for idx in range(n_discriminators): discriminator = discriminators[idx] for param in discriminator.parameters(): param.requires_grad = True sup_preds[idx] = sup_preds[idx].detach() d_outs[idx] = discriminators[idx](sup_preds[idx]) label_size = d_outs[idx].data.size() labels = torch.FloatTensor(label_size).fill_(source_label) labels = Variable(labels).cuda(args.gpu) d_losses.append(calc_bce_loss(d_outs[idx], labels)) d_losses[idx] = d_losses[idx] / args.iter_size / 2 d_losses[idx].backward() d_loss_vals[idx] += d_losses[idx].item() for idx in range(n_discriminators): stu_unsup_preds[idx] = stu_unsup_preds[idx].detach() d_outs[idx] = discriminators[idx](stu_unsup_preds[idx]) label_size = d_outs[idx].data.size() labels = torch.FloatTensor(label_size).fill_(tgt_label) labels = Variable(labels).cuda(args.gpu) d_losses[idx] = calc_bce_loss(d_outs[idx], labels) d_losses[idx] = d_losses[idx] / args.iter_size / 2 d_losses[idx].backward() d_loss_vals[idx] += d_losses[idx].item() for d_optimizer in d_optimizers: d_optimizer.step() total_loss = total_seg_loss + total_adv_loss + total_mse_loss total_loss.backward() student_optimizer.step() teacher_optimizer.step() log_str = 'iter = {0:7d}/{1:7d}'.format(i_iter, args.num_steps) log_str += ', total_seg_loss = {0:.3f} '.format(total_seg_loss) templ = 'seg_losses = [' + ', '.join(['%.2f'] * len(seg_loss_vals)) log_str += templ % tuple(seg_loss_vals) + '] ' templ = 'ens_losses = [' + ', '.join(['%.5f'] * len(unsup_loss_vals)) log_str += templ % tuple(unsup_loss_vals) + '] ' templ = 'adv_losses = [' + ', '.join(['%.2f'] * len(adv_tgt_loss_vals)) log_str += templ % tuple(adv_tgt_loss_vals) + '] ' templ = 'd_losses = [' + ', '.join(['%.2f'] * len(d_loss_vals)) log_str += templ % tuple(d_loss_vals) + '] ' print(log_str) if i_iter >= args.num_steps_stop - 1: print('save model ...') filename = 'UNet' + str( args.num_steps_stop) + '_v18_weightedclass.pth' torch.save(teacher_net.cpu().state_dict(), os.path.join(args.snapshot_dir, filename)) break if i_iter % args.save_pred_every == 0 and i_iter != 0: print('taking snapshot ...') filename = 'UNet' + str(i_iter) + '_v18_weightedclass.pth' torch.save(teacher_net.cpu().state_dict(), os.path.join(args.snapshot_dir, filename)) teacher_net.cuda(args.gpu)