def fit_demography(syn_sfs, args): ''' The second step in the procedure. Fits a dadi demographic model, with hard-coded assumptions about development and growth, to the synonymous mutation set. Uses the dadi_pipeline wrapper package. ''' pts = [args.samples + 10, args.samples + 20, args.samples + 30] #constrain optimization to Nu > 1 only. My initial round of 6k cells is definitely not shrinking. p_labels = "nu, T" upper = [1000, 1000] lower = [1, .001] Optimize_Functions.Optimize_Routine(syn_sfs, pts, str(args.samples), args.model_name, exponential_development, 3, 2, fs_folded=False, param_labels=p_labels, in_upper=upper, in_lower=lower)
# Below labels and upper and lower bounds of parameter values p_labels = "nu1, nu2, T1, m12, m21" upper = [20, 20, 10, 200, 200] lower = [1e-3, 1e-3, 1e-3, 1e-5, 1e-5] # Here details of the optimization routines: 4 rounds of optimization for large dataset and unfolded spectrum reps = [10, 10, 20, 20] maxiters = [10, 20, 20, 30] folds = [3, 3, 2, 1] # Run 10 Independent optimizations routines, at the end use the "Summarize_Output.py" to keep the best run from each optimization routine for i in range(1, 11): prefix = infile + "_OPTI_Number_{}".format(i) Optimize_Functions.Optimize_Routine(data, prefix, "IM", IM, 4, 5, data_folded=False, param_labels=p_labels, in_upper=upper, in_lower=lower, reps=reps, maxiters=maxiters, folds=folds)
Portik, D.M., Leache, A.D., Rivera, D., Blackburn, D.C., Rodel, M.-O., Barej, M.F., Hirschfeld, M., Burger, M., and M.K.Fujita. 2017. Evaluating mechanisms of diversification in a Guineo-Congolian forest frog using demographic model selection. Molecular Ecology 26: 5245-5263. doi: 10.1111/mec.14266 ''' # Split into two populations, no migration. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "no_mig", Models_2D.no_mig, rounds, 3, fs_folded=fs_folded, reps=reps, maxiters=maxiters, folds=folds, param_labels="nu1, nu2, T") # Split into two populations, with continuous symmetric migration. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "sym_mig", Models_2D.sym_mig, rounds, 4, fs_folded=fs_folded,
# These are the grid point settings will use for extrapolation. pts = [120, 160, 200] prefix = "optim_a/priorsize_asym_mig" p_labels = "nua, T1, nu1b, nu2b, T2, m12, m21" upper = [100, 10, 100, 100, 10, 100, 100] lower = [1e-2, 0, 1e-2, 1e-2, 0, 0, 0] p0 = [1, 1, 1, 1, 1, 1, 1] reps = [10, 10, 10, 10] maxiters = [5, 5, 10, 20] folds = [5, 3, 2, 1] i = sys.argv[1] Optimize_Functions.Optimize_Routine(fs, pts, prefix + "_{}".format(i), func.func_name, func, len(reps), len(p0), fs_folded=True, param_labels=p_labels, in_upper=upper, in_lower=lower, reps=reps, maxiters=maxiters, folds=folds)
''' Example 1. Now let's use the function to run an optimization routine for our data and this model. We need to specify the first seven arguments in this function, but there are other options we can also use if we wanted more control over the optimization scheme. We'll start with the basic version here. The argument explanations are above. This would perform three rounds of optimizations, using a default number of replicates for each round (see documentation for explanation of default values). ''' #create a prefix to label the output files prefix = "V1" #make sure to define your extrapolation grid size pts = [50,60,70] #Remember the order for mandatory arguments as below #Optimize_Routine(fs, pts, outfile, model_name, func, rounds, param_number, fs_folded) Optimize_Functions.Optimize_Routine(fs, pts, prefix, "sym_mig", sym_mig, 3, 4, fs_folded=True) ''' Example 2. It is a good idea to include the labels of the parameters so they can get written to the output file, otherwise you'll have to go back to the model each time you wanted to see their order. The optional arguments require using the = sign to assign a variable or value to the argument. ''' prefix = "V2" pts = [50,60,70] p_labels = "nu1, nu2, m, T" Optimize_Functions.Optimize_Routine(fs, pts, prefix, "sym_mig", sym_mig, 3, 4, fs_folded=True,
''' Diversification Model Set This first set of models come from the following publication: Portik, D.M., Leache, A.D., Rivera, D., Blackburn, D.C., Rodel, M.-O., Barej, M.F., Hirschfeld, M., Burger, M., and M.K. Fujita. 2017. Evaluating mechanisms of diversification in a Guineo-Congolian forest frog using demographic model selection. Molecular Ecology 26: 5245-5263. doi: 10.1111/mec.14266 ''' # Split into three populations, no migration. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "split_nomig", Models_3D.split_nomig, rounds, 6, fs_folded=fs_folded, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, T1, T2") # Split into three populations, symmetric migration between all populations (1<->2, 2<->3, and 1<->3). Optimize_Functions.Optimize_Routine(fs, pts, prefix, "split_symmig_all", Models_3D.split_symmig_all, rounds, 10, fs_folded=fs_folded, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, mA, m1, m2, m3, T1, T2") # Split into three populations, symmetric migration between 'adjacent' populations (1<->2, 2<->3, but not 1<->3). Optimize_Functions.Optimize_Routine(fs, pts, prefix, "split_symmig_adjacent", Models_3D.split_symmig_adjacent, rounds, 9, fs_folded=fs_folded, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, mA, m1, m2, m3, T1, T2") # Adjacent Secondary contact, longest isolation - Split between pop 1 and (2,3) with no migration, then split between pop 2 and 3 with no migration. Period of symmetric secondary contact occurs between adjacent populations (ie 1<->2, 2<->3, but not 1<->3) after all splits are complete. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "refugia_adj_1", Models_3D.refugia_adj_1, rounds, 9, fs_folded=fs_folded, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, m1, m2, T1, T2, T3") # Adjacent Secondary contact, shorter isolation - Split between pop 1 and (2,3), gene flow does not occur. Split between pop 2 and 3 occurs with gene flow. After appearance of 2 and 3, gene flow also occurs between 1 and 2. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "refugia_adj_2", Models_3D.refugia_adj_2, rounds, 8, fs_folded=fs_folded,
# 3 values no_mig_params = [0.357, 0.7876, 0.1456] #"sym_mig" #4 Values sym_mig_params = [0.6832, 1.4416, 0.6072, 2.5373] #"asym_mig" #5 Values asym_mig_params = [0.938, 3.558, 0.5284, 0.1714, 5.8838] #====================================================================================== # Call the function with the relevant arguments. # Divergence with no migration Optimize_Functions.Optimize_Round2(pts, fs, outfile, reps, maxiter, "no_mig", no_mig_params) # Split into two populations, with continuous symmetric migration. Optimize_Functions.Optimize_Round2(pts, fs, outfile, reps, maxiter, "sym_mig", sym_mig_params) # Split into two populations, with continuous asymmetric migration. Optimize_Functions.Optimize_Round2(pts, fs, outfile, reps, maxiter, "asym_mig", asym_mig_params) #=========================================================================== #clock the amount of time to complete the script t_finish = datetime.now() elapsed = t_finish - t_begin print '\n', '\n', "-----------------------------------------------------------------------------------------------------" print "Finished all analyses!"
pts = [30, 40, 50] #prefix for output file naming outfile = "C-O" #spectrum object name (we defined this above) fs = fs_1 #integer to control number of replicates per model reps = int(30) #max number of iterations per optimization step (though see dadi user group for explanation) maxiter = int(20) #====================================================================================== # Call the function with the relevant arguments. # 1 Standard neutral model, populations never diverge Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "no_divergence") # 2 Split into two populations, no migration. Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "no_mig") # 3 Split into two populations, with continuous symmetric migration. Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "sym_mig") # 4 Split into two populations, with continuous asymmetric migration. Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "asym_mig") #=========================================================================== #clock the amount of time to complete the script t_finish = datetime.now() elapsed = t_finish - t_begin print '\n', '\n', "-----------------------------------------------------------------------------------------------------"
#====================================================================================== # Now call the function with the relevant arguments. # There are many models to test here. A brief definition is given for each, but the actual # models are defined in the Models_2D.py script. The first 15 were implemented in Portik # et al. 2016 (doi: 10.1111/mec.14266), the following 9 are newer for various projects. # Here it is set up to call each model one by one sequentially, which could finish relatively quickly. # If it takes too long, create multiple verisions of this script, block out some models (use hashes or delete), # and execute one version for every core you have available. It will greatly speed up these steps, # and sometimes if extrapolations fail the script will crash too and this could prevent it from # happening too many times. # Standard neutral model, populations never diverge Optimize_Functions.Optimize_Round3(pts, fs, outfile, reps, maxiter, "no_divergence", no_divergence_params) # Split into two populations, no migration. Optimize_Functions.Optimize_Round3(pts, fs, outfile, reps, maxiter, "no_mig", no_mig_params) # Split into two populations, with continuous symmetric migration. Optimize_Functions.Optimize_Round3(pts, fs, outfile, reps, maxiter, "sym_mig", sym_mig_params) # Split into two populations, with continuous asymmetric migration. Optimize_Functions.Optimize_Round3(pts, fs, outfile, reps, maxiter, "asym_mig", asym_mig_params) # Split with continuous symmetric migration, followed by isolation. Optimize_Functions.Optimize_Round3(pts, fs, outfile, reps, maxiter,
summary_file = open("%s/results_summary/%s_%s_%s_%s_results_summary.txt" % (out_dir,taxa,is_folded,model,sfs_type), "a+") param_list = '\t'.join(map(str,param_labels[model].replace(" ","").split(","))) summary_file.write("sfs_type\trun_num\ttaxa\tmodel\tlog-likelihood\taic\ttheta\t%s\n" % (param_list)) summary_file.close() #Show grid points in case they need to be checked in the standard output file print "\n\nGrid points: {}".format(pts) print "\n\nBeginning optimization run {}".format(run_num) #Set filename prefix prefix = "%s/%s_%s_%s_run%s" % (out_dir,taxa,is_folded,sfs_type,run_num) #Run optimization run with dadi_pipeline if is_folded == "fold": params = Optimize_Functions.Optimize_Routine(fs, pts, prefix, model, func, len(reps), len(lower), fs_folded = True, param_labels = p_labels, reps = reps, folds = folds, in_params = in_params, in_upper = upper, in_lower = lower, maxiters = maxiters) elif is_folded == "unfold": params = Optimize_Functions.Optimize_Routine(fs, pts, prefix, model, func, len(reps), len(lower), fs_folded = False, param_labels = p_labels, reps = reps, folds = folds, in_params = in_params, in_upper = upper, in_lower = lower, maxiters = maxiters) else: print "\n\nSyntax incorrect for folding argument, please use 'fold' or 'unfold'" #Print notable results and where they are saved to. print "\n\nHighest likelihood of run: {}".format(params[0]) print "\nOptimized parameter values: {}".format(params[1]) print "\nAdding run results to analysis summary file: %s/results_summary/%s_%s_%s_%s_results_summary.txt" % (out_dir,taxa,is_folded,model,sfs_type) #Once again make sure summary file is there. if not os.path.isfile("%s/results_summary/%s_%s_%s_%s_results_summary.txt" % (out_dir,taxa,is_folded,model,sfs_type)): summary_file = open("%s/results_summary/%s_%s_%s_%s_results_summary.txt" % (out_dir,taxa,is_folded,model,sfs_type), "a+") param_list = '\t'.join(map(str,param_labels[model].replace(" ","").split(","))) summary_file.write("sfs_type\trun_num\ttaxa\tmodel\tlog-likelihood\taic\ttheta\t%s\n" % (param_list))
#prefix for output file naming outfile = "N_v_S" #These can be left alone, unless you want more searches: #spectrum object name (we defined this above) fs = fs_1 #=========================================================================== # Now call the function with the relevant arguments. # There are several models to optimize. # Each model is executed with one replicate using fixed parameter values. # The simulated model is stored as an object to be called on in the plotting function. # Standard neutral model, populations never diverge no_divergence = Optimize_Functions.Optimize_Single(pts, fs, "no_divergence", no_divergence_params) # Split into two populations, no migration. no_mig = Optimize_Functions.Optimize_Single(pts, fs, "no_mig", no_mig_params) # Split into two populations, with continuous symmetric migration. sym_mig = Optimize_Functions.Optimize_Single(pts, fs, "sym_mig", sym_mig_params) # Split into two populations, with continuous asymmetric migration. asym_mig = Optimize_Functions.Optimize_Single(pts, fs, "asym_mig", asym_mig_params) # Split with continuous symmetric migration, followed by isolation. anc_sym_mig = Optimize_Functions.Optimize_Single(pts, fs, "anc_sym_mig", anc_sym_mig_params)
fs_folded = True #************** # To change bounds, edit upper and lower. # To use default, comment out upper and lower and remove "in_upper=upper, in_lower=lower," from Optimize_Routine. # Split into two populations, no migration. upper = [30, 30, 30] lower = [1e-05, 1e-05, 1e-05] Optimize_Functions.Optimize_Routine(fs, prefix, "no_mig", Models_2D.no_mig, rounds, 3, fs_folded=fs_folded, reps=reps, maxiters=maxiters, folds=folds, in_upper=upper, in_lower=lower, param_labels="nu1, nu2, T") # Split into two populations, with continuous symmetric migration. upper = [30, 30, 30, 30] lower = [1e-05, 1e-05, 0, 1e-05] Optimize_Functions.Optimize_Routine(fs, prefix, "sym_mig", Models_2D.sym_mig, rounds,
#====================================================================================== # Now call the function with the relevant arguments. # There are many models to test here. A brief definition is given for each, but the actual # models are defined in the Models_2D.py script. The first 15 were implemented in Portik # et al. 2016 (doi: 10.1111/mec.14266), the following 9 are newer for various projects. # Here it is set up to call each model one by one sequentially, which could finish relatively quickly. # If it takes too long, create multiple verisions of this script, block out some models (use hashes or delete), # and execute one version for every core you have available. It will greatly speed up these steps, # and sometimes if extrapolations fail the script will crash too and this could prevent it from # happening too many times. # Standard neutral model, populations never diverge Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "no_divergence") # Split into two populations, no migration. Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "no_mig") # Split into two populations, with continuous symmetric migration. Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "sym_mig") # Split into two populations, with continuous asymmetric migration. Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "asym_mig") # Split with continuous symmetric migration, followed by isolation. Optimize_Functions.Optimize_Round1(pts, fs, outfile, reps, maxiter, "anc_sym_mig") # Split with continuous asymmetric migration, followed by isolation.
reps = [10, 20, 30, 40] maxiters = [3, 5, 10, 15] folds = [3, 2, 2, 1] fs_folded = True prefix = "Two_pop_" #no dovergence Optimize_Functions.Optimize_Routine(fs, pts, prefix, "no_divergence", Models_2D.no_divergence, rounds, 1, fs_folded=fs_folded, optimizer="log", maxiters=maxiters, folds=folds, param_labels="nu1") #Split into two populations, no migration. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "no_mig", Models_2D.no_mig, rounds, 3, fs_folded=fs_folded,
''' Diversification Model Set This first set of models come from the following publication: Portik, D.M., Leache, A.D., Rivera, D., Blackburn, D.C., Rodel, M.-O., Barej, M.F., Hirschfeld, M., Burger, M., and M.K. Fujita. 2017. Evaluating mechanisms of diversification in a Guineo-Congolian forest frog using demographic model selection. Molecular Ecology 26: 5245-5263. doi: 10.1111/mec.14266 ''' # Split into three populations, no migration. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "split_nomig", Models_3D.split_nomig, rounds, 6, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, T1, T2") # Split into three populations, symmetric migration between all populations (1<->2, 2<->3, and 1<->3). Optimize_Functions.Optimize_Routine(fs, pts, prefix, "split_symmig_all", Models_3D.split_symmig_all, rounds, 10, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, mA, m1, m2, m3, T1, T2") # Split into three populations, symmetric migration between 'adjacent' populations (1<->2, 2<->3, but not 1<->3). Optimize_Functions.Optimize_Routine(fs, pts, prefix, "split_symmig_adjacent", Models_3D.split_symmig_adjacent, rounds, 9, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, mA, m1, m2, m3, T1, T2") # Adjacent Secondary contact, longest isolation - Split between pop 1 and (2,3) with no migration, then split between pop 2 and 3 with no migration. Period of symmetric secondary contact occurs between adjacent populations (ie 1<->2, 2<->3, but not 1<->3) after all splits are complete. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "refugia_adj_1", Models_3D.refugia_adj_1, rounds, 9, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, m1, m2, T1, T2, T3") # Adjacent Secondary contact, shorter isolation - Split between pop 1 and (2,3), gene flow does not occur. Split between pop 2 and 3 occurs with gene flow. After appearance of 2 and 3, gene flow also occurs between 1 and 2. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "refugia_adj_2", Models_3D.refugia_adj_2, rounds, 8, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, m1, m2, T1, T2") # Adjacent Secondary contact, shortest isolation - Split between pop 1 and (2,3) with no migration. Split between pop 2 and 3 occurs with gene flow, and gene flow occurs between 1 and 2 as well. Optimize_Functions.Optimize_Routine(fs, pts, prefix, "refugia_adj_3", Models_3D.refugia_adj_3, rounds, 10, reps=reps, maxiters=maxiters, folds=folds, param_labels = "nu1, nuA, nu2, nu3, mA, m1, m2, T1a, T1b, T2")
pts = [120, 130, 140] p_labels = "nu1, nu2, m12, m21, T1,T2" upper = [20, 20, 10, 10, 5, 5] lower = [1e-3, 1e-3, 1e-5, 1e-5, 1e-3, 1e-3] # Here details of the optimization routines: 3 rounds of optimization for the simple IM and SC models reps = [10, 10, 10] maxiters = [10, 20, 30] folds = [3, 2, 1] # Run 3 Independent optimizations routines, at the end use the "Summarize_Output.py" to keep the best run from each optimization routine for i in range(1, 4): prefix = infile + "_OPTI_Number_{}".format(i) Optimize_Functions.Optimize_Routine(data, pts, prefix, "SC", SC, 3, 6, data_folded=False, param_labels=p_labels, in_upper=upper, in_lower=lower, reps=reps, maxiters=maxiters, folds=folds, optimizer="log")