def save_convtest(slf, diff, thresh, info, printout=True, extratxt=["", ""]): """ Save the details of what parameters have converged. """ msgs.info("Saving the best-fitting model parameters", verbose=slf._argflag['out']['verbose']) filename = extratxt[0] + slf._argflag['run'][ 'modname'] + '.conv' + extratxt[1] prestring = "#\n# Generated by ALIS on {0:s}\n#\n".format( datetime.datetime.now().strftime("%d/%m/%y at %H:%M:%S")) prestring += "# Running Time (hrs) = {0:f}\n".format(info[0]) prestring += "# Initial Chi-Squared = {0:f}\n".format(slf._chisq_init) prestring += "# Bestfit Chi-Squared = {0:f}\n".format(info[1]) prestring += "# Degrees-of-Freedom = {0:d}\n".format(info[2]) prestring += "# Num. of Iterations = {0:d}\n".format(info[3]) prestring += "# Convergence Reason = {0:s}\n".format( getreason(info[4], verbose=slf._argflag['out']['verbose'])) prestring += "\n" inputmodl = "#\n" for i in range(len(slf._parlines)): prestring += slf._parlines[i] inputmodl += "# " + slf._parlines[i] prestring += "\ndata read\n" inputmodl += "# data read\n" msgs.bug("INCLUDE THE BEST-FIT FWHM HERE --- PLACE THE ERROR BELOW", verbose=slf._argflag['out']['verbose']) for i in range(len(slf._datlines)): prestring += slf._datlines[i] inputmodl += "# " + slf._datlines[i] prestring += "data end\n" inputmodl += "# data end\n" prestring += "\nmodel read\n" inputmodl += "# model read\n" modcomlin = [] modcomind = [] tconvstring = '' for i in range(len(slf._modlines)): if slf._modlines[i].split()[0] in ["fix", "lim"]: tconvstring += slf._modlines[i].replace('\t', ' ') if slf._modlines[i].lstrip()[0] == '#': modcomlin.append(slf._modlines[i].rstrip('\n')) modcomind.append(i) inputmodl += "# " + slf._modlines[i] convstring = print_model(diff, slf._modpass, thresh, verbose=slf._argflag['out']['verbose'], funcarray=slf._funcarray) # if printout and slf._argflag['out']['verbose'] != -1: # print "\n####################################################" # print convstring # print "####################################################\n" # Reinsert the comments at the original locations cnvstrspl = (tconvstring + convstring).split('\n') for i in range(len(modcomlin)): cnvstrspl.insert(modcomind[i], modcomlin[i]) convstring = '\n'.join(cnvstrspl) # Include an end tag for the model convstring += "model end\n" inputmodl += "# model end\n#\n\n" if slf._argflag['out']['overwrite']: ans = 'y' else: ans = '' if os.path.exists(filename): while ans != 'y' and ans != 'n' and ans != 'r': msgs.warn("File %s exists!" % (filename), verbose=slf._argflag['out']['verbose']) ans = raw_input(msgs.input() + "Overwrite? (y/n) or rename? (r) - ") if ans == 'r': fileend = raw_input(msgs.input() + "Enter new filename - ") filename = fileend if os.path.exists(filename): ans = '' if ans != 'n': infile = open(filename, "w") infile.write(prestring) infile.write(convstring + "\n") infile.write("\n###################################################") infile.write("\n# #") infile.write("\n# HERE IS A COPY OF THE INPUT MODEL #") infile.write("\n# #") infile.write("\n###################################################\n") infile.write(inputmodl) infile.close() msgs.info("Saved output file successfully:" + msgs.newline() + filename, verbose=slf._argflag['out']['verbose'])
def perturb(slf, covar, bparams, parinfo): # Decide how many characters to use for output files nchr = str(np.int(np.log10(slf._argflag['sim']['perturb']))+1) # Create the directory structure for the simulations make_directory(slf._argflag['sim']['dirname'],overwrite=slf._argflag['out']['overwrite'],verbose=slf._argflag['out']['verbose']) # Grab the best-fitting model perror = np.sqrt(np.diag(covar)) # Store the starting parameters in an array outpert = np.array([slf._modpass['p0']]) wavf, fluf, errf = np.array([]), np.array([]), np.array([]) for sp in range(len(slf._posnfull)): wavf = np.append(wavf, slf._wavefit[sp].copy()) fluf = np.append(fluf, slf._fluxfit[sp].copy()) errf = np.append(errf, slf._fluefit[sp].copy()) # Find the non-zero elements in the covariance matrix cvsize = covar.shape[0] cxzero, cyzero = np.where(covar==0.0) bxzero, byzero = np.bincount(cxzero), np.bincount(cyzero) wxzero, wyzero = np.where(bxzero==cvsize)[0], np.where(byzero==cvsize)[0] zrocol = np.intersect1d(wxzero, wyzero) # This is the list of columns (or rows), where all elements are zero # Create a mask for the non-zero elements mask=np.zeros_like(covar) mask[:,zrocol],mask[zrocol,:]=1,1 cvnz=np.zeros((cvsize-zrocol.size,cvsize-zrocol.size)) cvnz[np.where(cvnz==0.0)]=covar[np.where(mask==0.0)] # Generate a new set of starting parameters from the covariance matrix X_covar_fit=np.matrix(np.random.standard_normal((cvnz.shape[0],slf._argflag['sim']['perturb']))) C_covar_fit=np.matrix(cvnz) U_covar_fit=np.linalg.cholesky(C_covar_fit) Y_covar_fit=U_covar_fit * X_covar_fit # Run through the simulations for sim in range(slf._argflag['sim']['perturb']): ntxt="{0:0"+nchr+"d}" ntxt=ntxt.format(sim+slf._argflag['sim']['startid']) # Text Identifier used as output msgs.test("PERTURB -- Realisation {0:s}/{1:s} began {2:s}".format(str(sim+1),str(slf._argflag['sim']['perturb']),time.ctime()),verbose=slf._argflag['out']['verbose']) # Enter the new starting parameters p0new = [] cntr=0 for pw in range(len(slf._modpass['p0'])): if perror[pw] > 0.0: # Check that the maximum level of perturbation has not been reached if slf._modpass['p0'][pw] != 0.0: if (Y_covar_fit[cntr,sim]).flatten()[0]/slf._modpass['p0'][pw] > slf._argflag['sim']['maxperturb']: p0new.append( slf._modpass['p0'][pw]*(1.0+slf._argflag['sim']['maxperturb']) ) cntr+=1 elif (Y_covar_fit[cntr,sim]).flatten()[0]/slf._modpass['p0'][pw] < -1.0*slf._argflag['sim']['maxperturb']: p0new.append( slf._modpass['p0'][pw]*(1.0-slf._argflag['sim']['maxperturb']) ) cntr+=1 else: p0new.append( slf._modpass['p0'][pw]+(Y_covar_fit[cntr,sim]).flatten()[0] ) cntr+=1 else: p0new.append( slf._modpass['p0'][pw]+(Y_covar_fit[cntr,sim]).flatten()[0] ) cntr+=1 else: p0new.append(slf._modpass['p0'][pw]) # Make sure all of the new parameters are within parinfo limits for i in range(len(p0new)): if parinfo[i]['limited'][0] == 1: if parinfo[i]['limits'][0] > p0new[i]: p0new[i] = parinfo[i]['limits'][0] if parinfo[i]['limited'][1] == 1: if parinfo[i]['limits'][1] < p0new[i]: p0new[i] = parinfo[i]['limits'][1] parinfo[i]['value'] = p0new[i] #alsave.save_model(slf, p0new, mfit.perror, [(0.0 - 0.0)/3600.0, mfit.fnorm, mfit.dof, mfit.niter, mfit.status], printout=False, extratxt=[slf._argflag['sim']['dirname']+'/',".new"]) #np.savetxt(slf._argflag['run']['modname']+'.covar',mfit.covar) fdict = slf.__dict__.copy() fa = {'x':wavf, 'y':fluf, 'err':errf, 'fdict':fdict} # Calculate the starting chi-squared start_func = myfunct_wrap(p0new,output=2,**fa) slf._chisq_init = np.sum(((fluf-start_func)/errf)**2) if np.isnan(slf._chisq_init): msgs.error("Initial chi-squared is not a number") if slf._chisq_init == np.Inf: msgs.error("Input chi-squared is Infinite"+msgs.newline()+"Perhaps the error spectrum is zero?") # Fit the realisation # msgs.info("Using {0:d} CPUs".format(slf._argflag['run']['ncpus']),verbose=slf._argflag['out']['verbose']) tstart=time.time() mr = alfit(myfunct_wrap, p0new, parinfo=parinfo, functkw=fa, verbose=1, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) # mr = alfit(myfunct_wrap, slf._modpass['p0'], parinfo=parinfo, functkw=fa, # verbose=0, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], # ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], # ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) tend=time.time() if mr.status <= 0: if mr.status == -20: msgs.info("Simulation was interrupted",verbose=slf._argflag['out']['verbose']) return else: msgs.error(mr.errmsg) else: msgs.info("Reason for convergence:"+msgs.newline()+alutils.getreason(mr.status,verbose=slf._argflag['out']['verbose']),verbose=slf._argflag['out']['verbose']) if mr.perror is None: msgs.bug("Errors returned from perturbed fit is None",verbose=slf._argflag['out']['verbose']) msgs.error("Cannot continue with the simulations") # Get the results and print them to file outpert = np.append(outpert, np.array([np.array(mr.params)]),axis=0) alsave.save_model(slf, mr.params, mr.perror, [(tend - tstart)/3600.0, mr.fnorm, mr.dof, mr.niter, mr.status], printout=False, extratxt=[slf._argflag['sim']['dirname']+'/',".perturb"+ntxt]) # Plot the data (if requested) if slf._argflag['plot']['fits']: model = myfunct_wrap(mr.params,output=3,**fa) alplot.make_plots_all(slf, model=model) fileend=raw_input(msgs.input()+"Press enter to view the fits -") alplot.plot_showall() ntxt=":0"+nchr+"d}" outname="{0:s}.{1:s}_{2"+ntxt+"-{3"+ntxt msgs.info("Saving the results from the simulations",verbose=slf._argflag['out']['verbose']) pertname=outname.format(slf._argflag['run']['modname'],'perturb',slf._argflag['sim']['startid'],slf._argflag['sim']['perturb']+slf._argflag['sim']['startid']) np.savetxt(pertname,outpert) return
def sim_systematics(slf, p0new, parinfo, ntxt, edgearr): """ This implementation of Sim Systematics uses the information in 'systematics' (passed in as input) to refit the continuum of the fake data. 'systematics' should be a keyword red in from the 'data read' section. ------------------------------- Systematic errors include: + choice of continuum + choice of starting parameters ------------------------------- """ # Make changes to the continuum wavf, fluf, errf = np.array([]), np.array([]), np.array([]) stf, enf = [0 for all in slf._posnfull], [0 for all in slf._posnfull] for sp in range(len(slf._posnfull)): for sn in range(len(slf._posnfull[sp])-1): ll = slf._posnfull[sp][sn] lu = slf._posnfull[sp][sn+1] if slf._datopt['systmodule'][sp][sn] is None: # Don't make any systematic corrections to these spectra msgs.warn("Not applying any systematic corrections to file:"+msgs.newline()+slf._snipnames[sp][sn],verbose=slf._argflag['out']['verbose']) newfluxfull, newfluefull = np.copy(slf._fluxfull[sp][ll:lu]), np.copy(slf._fluefull[sp][ll:lu]) elif slf._datopt['systmodule'][sp][sn] == 'default' or slf._datopt['systmodule'][sp][sn] == 'continuumpoly': # The default systematics routine --- fit a polynomial to the continuum newfluxfull, newfluefull = syst_continuumpoly(slf._wavefull[sp][ll:lu], slf._fluxfull[sp][ll:lu], slf._fluefull[sp][ll:lu], slf._systfull[sp][ll:lu], edgearr[sp][sn], slf._snipnames[sp][sn], verbose=slf._argflag['out']['verbose']) #elif slf._datopt['systmodule'][sp][sn] == other built-in function: else: # If the routine isn't built-in, it must be user-defined # Get the identifier text if ',' in slf._datopt['systmodule'][sp][sn]: filename, idtxt = slf._datopt['systmodule'][sp][sn].split(',') else: filename, idtxt = slf._datopt['systmodule'][sp][sn], 'systmodule' path, file = os.path.split(filename) name, ext = os.path.splitext(file) # Set the import loader impload = srcloader() modu = impload.find_module_in_dir(name, path) if not modu: msgs.error("Could not import {0:s}".format(name)) usrsystmod = impload.load_module(name, modu) newfluxfull, newfluefull = usrsystmod.loader(idtxt, slf._wavefull[sp][ll:lu], slf._fluxfull[sp][ll:lu], slf._fluefull[sp][ll:lu], slf._systfull[sp][ll:lu], edgearr[sp][sn], slf._snipnames[sp][sn]) # Assign the adjusted data to the slf class (and multiply by the user-specified continuum). slf._fluxfull[sp][ll:lu], slf._fluefull[sp][ll:lu] = copy.deepcopy(newfluxfull)*slf._contfull[sp][ll:lu], copy.deepcopy(newfluefull)*slf._contfull[sp][ll:lu] # Make the appropriate changes to the fitted spectral region w = np.where((slf._wavefull[sp][ll:lu] >= slf._posnfit[sp][2*sn+0]) & (slf._wavefull[sp][ll:lu] <= slf._posnfit[sp][2*sn+1])) wA= np.in1d(slf._wavefull[sp][ll:lu][w], slf._wavefit[sp]) wB= np.where(wA==True) enf[sp] = stf[sp] + slf._wavefull[sp][ll:lu][w][wB].size slf._fluxfit[sp][stf[sp]:enf[sp]] = slf._fluxfull[sp][ll:lu][w][wB] slf._fluefit[sp][stf[sp]:enf[sp]] = slf._fluefull[sp][ll:lu][w][wB] stf[sp] = enf[sp] # Place the fitted regions into a single array to be read by th echi-squared fitting program wavf = np.append(wavf, slf._wavefit[sp]) fluf = np.append(fluf, slf._fluxfit[sp]) errf = np.append(errf, slf._fluefit[sp]) # Make sure all of the new parameters are within parinfo limits for i in range(len(p0new)): if parinfo[i]['limited'][0] == 1: if parinfo[i]['limits'][0] > p0new[i]: p0new[i] = parinfo[i]['limits'][0] if parinfo[i]['limited'][1] == 1: if parinfo[i]['limits'][1] < p0new[i]: p0new[i] = parinfo[i]['limits'][1] parinfo[i]['value'] = p0new[i] # Update functargs and fit the data! fdict = slf.__dict__.copy() fa = {'x':wavf, 'y':fluf, 'err':errf, 'fdict':fdict} # Calculate the starting chi-squared start_func = myfunct_wrap(p0new,output=2,**fa) slf._chisq_init = np.sum(((fluf-start_func)/errf)**2) if np.isnan(slf._chisq_init): msgs.error("Initial chi-squared is not a number") if slf._chisq_init == np.Inf: msgs.error("Input chi-squared is Infinite"+msgs.newline()+"Perhaps the error spectrum is zero?") if slf._argflag['plot']['fits']: model = myfunct_wrap(p0new,output=3,**fa) alplot.make_plots_all(slf,model=model) fileend=raw_input(msgs.input()+"Press enter to view the fits -") alplot.plot_showall() # Now fit it! tstart=time.time() ms = alfit(myfunct_wrap, p0new, parinfo=parinfo, functkw=fa, verbose=1, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) tend=time.time() if ms.status <= 0: if ms.status == -20: msgs.info("Systematics simulation was interrupted",verbose=slf._argflag['out']['verbose']) return else: msgs.error(ms.errmsg) else: msgs.info("Reason for convergence:"+msgs.newline()+alutils.getreason(ms.status,verbose=slf._argflag['out']['verbose']),verbose=slf._argflag['out']['verbose']) if ms.perror is None: msgs.bug("Errors returned from systematics fit is None",verbose=slf._argflag['out']['verbose']) msgs.error("Cannot continue with the simulations") # Plot the data (if requested) if slf._argflag['plot']['fits']: model = myfunct_wrap(ms.params,output=3,**fa) alplot.make_plots_all(slf,model=model) fileend=raw_input(msgs.input()+"Press enter to view the fits -") alplot.plot_showall() # Get the results and print them to file alsave.save_model(slf, ms.params, ms.perror, [(tend - tstart)/3600.0, ms.fnorm, ms.dof, ms.niter, ms.status], printout=False, extratxt=[slf._argflag['sim']['dirname']+'/',".syst"+ntxt]) return ms
def sim_systematics(slf, p0new, parinfo, ntxt, edgearr): """ This implementation of Sim Systematics uses the information in 'systematics' (passed in as input) to refit the continuum of the fake data. 'systematics' should be a keyword red in from the 'data read' section. ------------------------------- Systematic errors include: + choice of continuum + choice of starting parameters ------------------------------- """ # Make changes to the continuum wavf, fluf, errf = np.array([]), np.array([]), np.array([]) stf, enf = [0 for all in slf._posnfull], [0 for all in slf._posnfull] for sp in range(len(slf._posnfull)): for sn in range(len(slf._posnfull[sp]) - 1): ll = slf._posnfull[sp][sn] lu = slf._posnfull[sp][sn + 1] if slf._datopt['systmodule'][sp][sn] is None: # Don't make any systematic corrections to these spectra msgs.warn("Not applying any systematic corrections to file:" + msgs.newline() + slf._snipnames[sp][sn], verbose=slf._argflag['out']['verbose']) newfluxfull, newfluefull = np.copy( slf._fluxfull[sp][ll:lu]), np.copy( slf._fluefull[sp][ll:lu]) elif slf._datopt['systmodule'][sp][sn] == 'default' or slf._datopt[ 'systmodule'][sp][sn] == 'continuumpoly': # The default systematics routine --- fit a polynomial to the continuum newfluxfull, newfluefull = syst_continuumpoly( slf._wavefull[sp][ll:lu], slf._fluxfull[sp][ll:lu], slf._fluefull[sp][ll:lu], slf._systfull[sp][ll:lu], edgearr[sp][sn], slf._snipnames[sp][sn], verbose=slf._argflag['out']['verbose']) #elif slf._datopt['systmodule'][sp][sn] == other built-in function: else: # If the routine isn't built-in, it must be user-defined # Get the identifier text if ',' in slf._datopt['systmodule'][sp][sn]: filename, idtxt = slf._datopt['systmodule'][sp][sn].split( ',') else: filename, idtxt = slf._datopt['systmodule'][sp][ sn], 'systmodule' path, file = os.path.split(filename) name, ext = os.path.splitext(file) # Set the import loader impload = srcloader() modu = impload.find_module_in_dir(name, path) if not modu: msgs.error("Could not import {0:s}".format(name)) usrsystmod = impload.load_module(name, modu) newfluxfull, newfluefull = usrsystmod.loader( idtxt, slf._wavefull[sp][ll:lu], slf._fluxfull[sp][ll:lu], slf._fluefull[sp][ll:lu], slf._systfull[sp][ll:lu], edgearr[sp][sn], slf._snipnames[sp][sn]) # Assign the adjusted data to the slf class (and multiply by the user-specified continuum). slf._fluxfull[sp][ll:lu], slf._fluefull[sp][ll:lu] = copy.deepcopy( newfluxfull) * slf._contfull[sp][ll:lu], copy.deepcopy( newfluefull) * slf._contfull[sp][ll:lu] # Make the appropriate changes to the fitted spectral region w = np.where( (slf._wavefull[sp][ll:lu] >= slf._posnfit[sp][2 * sn + 0]) & (slf._wavefull[sp][ll:lu] <= slf._posnfit[sp][2 * sn + 1])) wA = np.in1d(slf._wavefull[sp][ll:lu][w], slf._wavefit[sp]) wB = np.where(wA == True) enf[sp] = stf[sp] + slf._wavefull[sp][ll:lu][w][wB].size slf._fluxfit[sp][stf[sp]:enf[sp]] = slf._fluxfull[sp][ll:lu][w][wB] slf._fluefit[sp][stf[sp]:enf[sp]] = slf._fluefull[sp][ll:lu][w][wB] stf[sp] = enf[sp] # Place the fitted regions into a single array to be read by th echi-squared fitting program wavf = np.append(wavf, slf._wavefit[sp]) fluf = np.append(fluf, slf._fluxfit[sp]) errf = np.append(errf, slf._fluefit[sp]) # Make sure all of the new parameters are within parinfo limits for i in range(len(p0new)): if parinfo[i]['limited'][0] == 1: if parinfo[i]['limits'][0] > p0new[i]: p0new[i] = parinfo[i]['limits'][0] if parinfo[i]['limited'][1] == 1: if parinfo[i]['limits'][1] < p0new[i]: p0new[i] = parinfo[i]['limits'][1] parinfo[i]['value'] = p0new[i] # Update functargs and fit the data! fdict = slf.__dict__.copy() fa = {'x': wavf, 'y': fluf, 'err': errf, 'fdict': fdict} # Calculate the starting chi-squared start_func = myfunct_wrap(p0new, output=2, **fa) slf._chisq_init = np.sum(((fluf - start_func) / errf)**2) if np.isnan(slf._chisq_init): msgs.error("Initial chi-squared is not a number") if slf._chisq_init == np.Inf: msgs.error("Input chi-squared is Infinite" + msgs.newline() + "Perhaps the error spectrum is zero?") if slf._argflag['plot']['fits']: model = myfunct_wrap(p0new, output=3, **fa) alplot.make_plots_all(slf, model=model) fileend = raw_input(msgs.input() + "Press enter to view the fits -") alplot.plot_showall() # Now fit it! tstart = time.time() ms = alfit(myfunct_wrap, p0new, parinfo=parinfo, functkw=fa, verbose=1, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) tend = time.time() if ms.status <= 0: if ms.status == -20: msgs.info("Systematics simulation was interrupted", verbose=slf._argflag['out']['verbose']) return else: msgs.error(ms.errmsg) else: msgs.info("Reason for convergence:" + msgs.newline() + alutils.getreason(ms.status, verbose=slf._argflag['out']['verbose']), verbose=slf._argflag['out']['verbose']) if ms.perror is None: msgs.bug("Errors returned from systematics fit is None", verbose=slf._argflag['out']['verbose']) msgs.error("Cannot continue with the simulations") # Plot the data (if requested) if slf._argflag['plot']['fits']: model = myfunct_wrap(ms.params, output=3, **fa) alplot.make_plots_all(slf, model=model) fileend = raw_input(msgs.input() + "Press enter to view the fits -") alplot.plot_showall() # Get the results and print them to file alsave.save_model( slf, ms.params, ms.perror, [(tend - tstart) / 3600.0, ms.fnorm, ms.dof, ms.niter, ms.status], printout=False, extratxt=[slf._argflag['sim']['dirname'] + '/', ".syst" + ntxt]) return ms
def sim_random(slf, covar, bparams, parinfo): # Decide how many characters to use for output files nchr = str(np.int(np.log10(slf._argflag['sim']['random']))+1) # Create the directory structure for the simulations make_directory(slf._argflag['sim']['dirname'],overwrite=slf._argflag['out']['overwrite'],verbose=slf._argflag['out']['verbose']) # Grab the best-fitting model perror = np.sqrt(np.diag(covar)) modlt = copy.deepcopy(slf._modconv_all) outrand, outsyst = np.array([slf._modpass['p0']]), np.array([slf._modpass['p0']]) # Store the starting parameters in an array wavf, errf = np.array([]), np.array([]) fluefull, fluefit = copy.deepcopy(slf._fluefull), copy.deepcopy(slf._fluefit) # Check for edge effects due to convolution and create the new wave and error arrays. # First find out which indices correspond to convolution cvind=[] for i in range(len(slf._modpass['emab'])): if slf._modpass['emab'][i] == 'cv': cvind.append(i) edgearr=[[] for i in range(len(slf._posnfit))] iind=0 for sp in range(len(slf._posnfull)): for sn in range(len(slf._posnfull[sp])-1): ll = slf._posnfull[sp][sn] lu = slf._posnfull[sp][sn+1] getstdd=[slf._argflag['sim']['edgecut'],slf._wavefull[sp][ll],slf._wavefull[sp][lu-1]] mtyp = slf._modpass['mtyp'][cvind[iind]] slf._funcarray[2][mtyp]._keywd = slf._modpass['mkey'][cvind[iind]] wvl, wvu = slf._funcarray[1][mtyp].set_vars(slf._funcarray[2][mtyp], bparams, slf._levadd[cvind[iind]], slf._modpass, cvind[iind], getstdd=getstdd) if wvl > slf._posnfit[sp][2*sn+0] or wvu < slf._posnfit[sp][2*sn+1]: msgs.warn("The random simulations cannot be trusted. The fitted is"+msgs.newline()+ "affected by edge effects from convolution. It is recommended"+msgs.newline()+ "that you input more data outside the fitted regions for:"+msgs.newline()+ slf._snipnames[sp][sn],verbose=slf._argflag['out']['verbose']) edgearr[sp].append([wvl,wvu]) iind += 1 wavf = np.append(wavf, slf._wavefit[sp].copy()) errf = np.append(errf, slf._fluefit[sp].copy()) # Find the non-zero elements in the covariance matrix cvsize = covar.shape[0] cxzero, cyzero = np.where(covar==0.0) bxzero, byzero = np.bincount(cxzero), np.bincount(cyzero) wxzero, wyzero = np.where(bxzero==cvsize)[0], np.where(byzero==cvsize)[0] zrocol = np.intersect1d(wxzero, wyzero) # This is the list of columns (or rows), where all elements are zero # Create a mask for the non-zero elements mask=np.zeros_like(covar) mask[:,zrocol],mask[zrocol,:]=1,1 cvnz=np.zeros((cvsize-zrocol.size,cvsize-zrocol.size)) cvnz[np.where(cvnz==0.0)]=covar[np.where(mask==0.0)] # Generate a new set of starting parameters from the covariance matrix if slf._argflag['sim']['newstart']: X_covar_fit=np.matrix(np.random.standard_normal((cvnz.shape[0],slf._argflag['sim']['random']))) C_covar_fit=np.matrix(cvnz) U_covar_fit=np.linalg.cholesky(C_covar_fit) Y_covar_fit=U_covar_fit * X_covar_fit # Run through the simulations for sim in range(slf._argflag['sim']['random']): ntxt="{0:0"+nchr+"d}" ntxt=ntxt.format(sim+slf._argflag['sim']['startid']) # Text Identifier used as output msgs.test("RANDOM ERRORS -- Realisation {0:s}/{1:s} began {2:s}".format(str(sim+1),str(slf._argflag['sim']['random']),time.ctime()),verbose=slf._argflag['out']['verbose']) # Generate a random realisation newfluxfull, newfluxfit = [], [] fluf = np.array([]) for sp in range(len(slf._posnfull)): newfluxfull.append(np.random.normal(modlt[sp],fluefull[sp])) newfluxfit.append(np.array([])) for sn in range(len(slf._posnfull[sp])-1): ll = slf._posnfull[sp][sn] lu = slf._posnfull[sp][sn+1] w = np.where((slf._wavefull[sp][ll:lu] >= slf._posnfit[sp][2*sn+0]) & (slf._wavefull[sp][ll:lu] <= slf._posnfit[sp][2*sn+1])) wA= np.in1d(slf._wavefull[sp][ll:lu][w], slf._wavefit[sp]) wB= np.where(wA==True) newfluxfit[sp] = np.append(newfluxfit[sp], np.copy(newfluxfull[sp][ll:lu][w][wB])) fluf = np.append(fluf, np.copy(newfluxfull[sp][ll:lu][w][wB])) slf._fluxfull, slf._fluxfit = copy.deepcopy(newfluxfull), copy.deepcopy(newfluxfit) slf._fluefull, slf._fluefit = copy.deepcopy(fluefull), copy.deepcopy(fluefit) p0new = [] if slf._argflag['sim']['newstart']: cntr=0 for pw in range(len(slf._modpass['p0'])): if perror[pw] > 0.0: p0new.append( slf._modpass['p0'][pw]+(Y_covar_fit[cntr,sim]).flatten()[0] ) cntr+=1 else: p0new.append(slf._modpass['p0'][pw]) else: p0new = slf._modpass['p0'] #alsave.save_model(slf, p0new, mfit.perror, [(0.0 - 0.0)/3600.0, mfit.fnorm, mfit.dof, mfit.niter, mfit.status], printout=False, extratxt=[slf._argflag['sim']['dirname']+'/',".new"]) #np.savetxt(slf._argflag['run']['modname']+'.covar',mfit.covar) fdict = slf.__dict__.copy() fa = {'x':wavf, 'y':fluf, 'err':errf, 'fdict':fdict} # Calculate the starting chi-squared start_func = myfunct_wrap(slf._modpass['p0'],output=2,**fa) slf._chisq_init = np.sum(((fluf-start_func)/errf)**2) if np.isnan(slf._chisq_init): msgs.error("Initial chi-squared is not a number") if slf._chisq_init == np.Inf: msgs.error("Input chi-squared is Infinite"+msgs.newline()+"Perhaps the error spectrum is zero?") # Fit the realisation # msgs.info("Using {0:d} CPUs".format(slf._argflag['run']['ncpus']),verbose=slf._argflag['out']['verbose']) tstart=time.time() mr = alfit(myfunct_wrap, slf._modpass['p0'], parinfo=parinfo, functkw=fa, verbose=1, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) # mr = alfit(myfunct_wrap, slf._modpass['p0'], parinfo=parinfo, functkw=fa, # verbose=0, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], # ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], # ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) tend=time.time() if mr.status <= 0: if mr.status == -20: msgs.info("Random simulation was interrupted",verbose=slf._argflag['out']['verbose']) return else: msgs.error(mr.errmsg) else: msgs.info("Reason for convergence:"+msgs.newline()+alutils.getreason(mr.status,verbose=slf._argflag['out']['verbose']),verbose=slf._argflag['out']['verbose']) if mr.perror is None: msgs.bug("Errors returned from random fit is None",verbose=slf._argflag['out']['verbose']) msgs.error("Cannot continue with the simulations") # Get the results and print them to file outrand = np.append(outrand, np.array([np.array(mr.params)]),axis=0) alsave.save_model(slf, mr.params, mr.perror, [(tend - tstart)/3600.0, mr.fnorm, mr.dof, mr.niter, mr.status], printout=False, extratxt=[slf._argflag['sim']['dirname']+'/',".rand"+ntxt]) # Plot the data (if requested) if slf._argflag['plot']['fits']: model = myfunct_wrap(mr.params,output=3,**fa) alplot.make_plots_all(slf, model=model) fileend=raw_input(msgs.input()+"Press enter to view the fits -") alplot.plot_showall() if slf._argflag['sim']['systematics']: # Calculate the systematics msgs.test("SYSTEMATIC ERRORS -- Realisation {0:s}/{1:s} began {2:s}".format(str(sim+1),str(slf._argflag['sim']['random']),time.ctime()),verbose=slf._argflag['out']['verbose']) ms = sim_systematics(slf, p0new, parinfo, ntxt, edgearr) outsyst = np.append(outsyst, np.array([np.array(ms.params)]),axis=0) ntxt=":0"+nchr+"d}" outname="{0:s}.{1:s}_{2"+ntxt+"-{3"+ntxt msgs.info("Saving the results from the random simulations",verbose=slf._argflag['out']['verbose']) randname=outname.format(slf._argflag['run']['modname'],'rand',slf._argflag['sim']['startid'],slf._argflag['sim']['random']+slf._argflag['sim']['startid']) np.savetxt(randname,outrand) if slf._argflag['sim']['systematics']: msgs.info("Saving the results from the systematics simulations",verbose=slf._argflag['out']['verbose']) systname=outname.format(slf._argflag['run']['modname'],'syst',slf._argflag['sim']['startid'],slf._argflag['sim']['random']+slf._argflag['sim']['startid']) np.savetxt(systname,outsyst) return
def perturb(slf, covar, bparams, parinfo): # Decide how many characters to use for output files nchr = str(np.int(np.log10(slf._argflag['sim']['perturb'])) + 1) # Create the directory structure for the simulations make_directory(slf._argflag['sim']['dirname'], overwrite=slf._argflag['out']['overwrite'], verbose=slf._argflag['out']['verbose']) # Grab the best-fitting model perror = np.sqrt(np.diag(covar)) # Store the starting parameters in an array outpert = np.array([slf._modpass['p0']]) wavf, fluf, errf = np.array([]), np.array([]), np.array([]) for sp in range(len(slf._posnfull)): wavf = np.append(wavf, slf._wavefit[sp].copy()) fluf = np.append(fluf, slf._fluxfit[sp].copy()) errf = np.append(errf, slf._fluefit[sp].copy()) # Find the non-zero elements in the covariance matrix cvsize = covar.shape[0] cxzero, cyzero = np.where(covar == 0.0) bxzero, byzero = np.bincount(cxzero), np.bincount(cyzero) wxzero, wyzero = np.where(bxzero == cvsize)[0], np.where( byzero == cvsize)[0] zrocol = np.intersect1d( wxzero, wyzero ) # This is the list of columns (or rows), where all elements are zero # Create a mask for the non-zero elements mask = np.zeros_like(covar) mask[:, zrocol], mask[zrocol, :] = 1, 1 cvnz = np.zeros((cvsize - zrocol.size, cvsize - zrocol.size)) cvnz[np.where(cvnz == 0.0)] = covar[np.where(mask == 0.0)] # Generate a new set of starting parameters from the covariance matrix X_covar_fit = np.matrix( np.random.standard_normal( (cvnz.shape[0], slf._argflag['sim']['perturb']))) C_covar_fit = np.matrix(cvnz) U_covar_fit = np.linalg.cholesky(C_covar_fit) Y_covar_fit = U_covar_fit * X_covar_fit # Run through the simulations for sim in range(slf._argflag['sim']['perturb']): ntxt = "{0:0" + nchr + "d}" ntxt = ntxt.format( sim + slf._argflag['sim']['startid']) # Text Identifier used as output msgs.test("PERTURB -- Realisation {0:s}/{1:s} began {2:s}".format( str(sim + 1), str(slf._argflag['sim']['perturb']), time.ctime()), verbose=slf._argflag['out']['verbose']) # Enter the new starting parameters p0new = [] cntr = 0 for pw in range(len(slf._modpass['p0'])): if perror[pw] > 0.0: # Check that the maximum level of perturbation has not been reached if slf._modpass['p0'][pw] != 0.0: if (Y_covar_fit[cntr, sim]).flatten()[0] / slf._modpass[ 'p0'][pw] > slf._argflag['sim']['maxperturb']: p0new.append(slf._modpass['p0'][pw] * (1.0 + slf._argflag['sim']['maxperturb'])) cntr += 1 elif (Y_covar_fit[cntr, sim] ).flatten()[0] / slf._modpass['p0'][ pw] < -1.0 * slf._argflag['sim']['maxperturb']: p0new.append(slf._modpass['p0'][pw] * (1.0 - slf._argflag['sim']['maxperturb'])) cntr += 1 else: p0new.append(slf._modpass['p0'][pw] + (Y_covar_fit[cntr, sim]).flatten()[0]) cntr += 1 else: p0new.append(slf._modpass['p0'][pw] + (Y_covar_fit[cntr, sim]).flatten()[0]) cntr += 1 else: p0new.append(slf._modpass['p0'][pw]) # Make sure all of the new parameters are within parinfo limits for i in range(len(p0new)): if parinfo[i]['limited'][0] == 1: if parinfo[i]['limits'][0] > p0new[i]: p0new[i] = parinfo[i]['limits'][0] if parinfo[i]['limited'][1] == 1: if parinfo[i]['limits'][1] < p0new[i]: p0new[i] = parinfo[i]['limits'][1] parinfo[i]['value'] = p0new[i] #alsave.save_model(slf, p0new, mfit.perror, [(0.0 - 0.0)/3600.0, mfit.fnorm, mfit.dof, mfit.niter, mfit.status], printout=False, extratxt=[slf._argflag['sim']['dirname']+'/',".new"]) #np.savetxt(slf._argflag['run']['modname']+'.covar',mfit.covar) fdict = slf.__dict__.copy() fa = {'x': wavf, 'y': fluf, 'err': errf, 'fdict': fdict} # Calculate the starting chi-squared start_func = myfunct_wrap(p0new, output=2, **fa) slf._chisq_init = np.sum(((fluf - start_func) / errf)**2) if np.isnan(slf._chisq_init): msgs.error("Initial chi-squared is not a number") if slf._chisq_init == np.Inf: msgs.error("Input chi-squared is Infinite" + msgs.newline() + "Perhaps the error spectrum is zero?") # Fit the realisation # msgs.info("Using {0:d} CPUs".format(slf._argflag['run']['ncpus']),verbose=slf._argflag['out']['verbose']) tstart = time.time() mr = alfit(myfunct_wrap, p0new, parinfo=parinfo, functkw=fa, verbose=1, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) # mr = alfit(myfunct_wrap, slf._modpass['p0'], parinfo=parinfo, functkw=fa, # verbose=0, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], # ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], # ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) tend = time.time() if mr.status <= 0: if mr.status == -20: msgs.info("Simulation was interrupted", verbose=slf._argflag['out']['verbose']) return else: msgs.error(mr.errmsg) else: msgs.info("Reason for convergence:" + msgs.newline() + alutils.getreason( mr.status, verbose=slf._argflag['out']['verbose']), verbose=slf._argflag['out']['verbose']) if mr.perror is None: msgs.bug("Errors returned from perturbed fit is None", verbose=slf._argflag['out']['verbose']) msgs.error("Cannot continue with the simulations") # Get the results and print them to file outpert = np.append(outpert, np.array([np.array(mr.params)]), axis=0) alsave.save_model( slf, mr.params, mr.perror, [(tend - tstart) / 3600.0, mr.fnorm, mr.dof, mr.niter, mr.status], printout=False, extratxt=[slf._argflag['sim']['dirname'] + '/', ".perturb" + ntxt]) # Plot the data (if requested) if slf._argflag['plot']['fits']: model = myfunct_wrap(mr.params, output=3, **fa) alplot.make_plots_all(slf, model=model) fileend = raw_input(msgs.input() + "Press enter to view the fits -") alplot.plot_showall() ntxt = ":0" + nchr + "d}" outname = "{0:s}.{1:s}_{2" + ntxt + "-{3" + ntxt msgs.info("Saving the results from the simulations", verbose=slf._argflag['out']['verbose']) pertname = outname.format( slf._argflag['run']['modname'], 'perturb', slf._argflag['sim']['startid'], slf._argflag['sim']['perturb'] + slf._argflag['sim']['startid']) np.savetxt(pertname, outpert) return
def sim_random(slf, covar, bparams, parinfo): # Decide how many characters to use for output files nchr = str(np.int(np.log10(slf._argflag['sim']['random'])) + 1) # Create the directory structure for the simulations make_directory(slf._argflag['sim']['dirname'], overwrite=slf._argflag['out']['overwrite'], verbose=slf._argflag['out']['verbose']) # Grab the best-fitting model perror = np.sqrt(np.diag(covar)) modlt = copy.deepcopy(slf._modconv_all) outrand, outsyst = np.array([slf._modpass['p0']]), np.array( [slf._modpass['p0']]) # Store the starting parameters in an array wavf, errf = np.array([]), np.array([]) fluefull, fluefit = copy.deepcopy(slf._fluefull), copy.deepcopy( slf._fluefit) # Check for edge effects due to convolution and create the new wave and error arrays. # First find out which indices correspond to convolution cvind = [] for i in range(len(slf._modpass['emab'])): if slf._modpass['emab'][i] == 'cv': cvind.append(i) edgearr = [[] for i in range(len(slf._posnfit))] iind = 0 for sp in range(len(slf._posnfull)): for sn in range(len(slf._posnfull[sp]) - 1): ll = slf._posnfull[sp][sn] lu = slf._posnfull[sp][sn + 1] getstdd = [ slf._argflag['sim']['edgecut'], slf._wavefull[sp][ll], slf._wavefull[sp][lu - 1] ] mtyp = slf._modpass['mtyp'][cvind[iind]] slf._funcarray[2][mtyp]._keywd = slf._modpass['mkey'][cvind[iind]] wvl, wvu = slf._funcarray[1][mtyp].set_vars( slf._funcarray[2][mtyp], bparams, slf._levadd[cvind[iind]], slf._modpass, cvind[iind], getstdd=getstdd) if wvl > slf._posnfit[sp][2 * sn + 0] or wvu < slf._posnfit[sp][2 * sn + 1]: msgs.warn( "The random simulations cannot be trusted. The fitted is" + msgs.newline() + "affected by edge effects from convolution. It is recommended" + msgs.newline() + "that you input more data outside the fitted regions for:" + msgs.newline() + slf._snipnames[sp][sn], verbose=slf._argflag['out']['verbose']) edgearr[sp].append([wvl, wvu]) iind += 1 wavf = np.append(wavf, slf._wavefit[sp].copy()) errf = np.append(errf, slf._fluefit[sp].copy()) # Find the non-zero elements in the covariance matrix cvsize = covar.shape[0] cxzero, cyzero = np.where(covar == 0.0) bxzero, byzero = np.bincount(cxzero), np.bincount(cyzero) wxzero, wyzero = np.where(bxzero == cvsize)[0], np.where( byzero == cvsize)[0] zrocol = np.intersect1d( wxzero, wyzero ) # This is the list of columns (or rows), where all elements are zero # Create a mask for the non-zero elements mask = np.zeros_like(covar) mask[:, zrocol], mask[zrocol, :] = 1, 1 cvnz = np.zeros((cvsize - zrocol.size, cvsize - zrocol.size)) cvnz[np.where(cvnz == 0.0)] = covar[np.where(mask == 0.0)] # Generate a new set of starting parameters from the covariance matrix if slf._argflag['sim']['newstart']: X_covar_fit = np.matrix( np.random.standard_normal( (cvnz.shape[0], slf._argflag['sim']['random']))) C_covar_fit = np.matrix(cvnz) U_covar_fit = np.linalg.cholesky(C_covar_fit) Y_covar_fit = U_covar_fit * X_covar_fit # Run through the simulations for sim in range(slf._argflag['sim']['random']): ntxt = "{0:0" + nchr + "d}" ntxt = ntxt.format( sim + slf._argflag['sim']['startid']) # Text Identifier used as output msgs.test( "RANDOM ERRORS -- Realisation {0:s}/{1:s} began {2:s}".format( str(sim + 1), str(slf._argflag['sim']['random']), time.ctime()), verbose=slf._argflag['out']['verbose']) # Generate a random realisation newfluxfull, newfluxfit = [], [] fluf = np.array([]) for sp in range(len(slf._posnfull)): newfluxfull.append(np.random.normal(modlt[sp], fluefull[sp])) newfluxfit.append(np.array([])) for sn in range(len(slf._posnfull[sp]) - 1): ll = slf._posnfull[sp][sn] lu = slf._posnfull[sp][sn + 1] w = np.where( (slf._wavefull[sp][ll:lu] >= slf._posnfit[sp][2 * sn + 0]) & (slf._wavefull[sp][ll:lu] <= slf._posnfit[sp][2 * sn + 1])) wA = np.in1d(slf._wavefull[sp][ll:lu][w], slf._wavefit[sp]) wB = np.where(wA == True) newfluxfit[sp] = np.append( newfluxfit[sp], np.copy(newfluxfull[sp][ll:lu][w][wB])) fluf = np.append(fluf, np.copy(newfluxfull[sp][ll:lu][w][wB])) slf._fluxfull, slf._fluxfit = copy.deepcopy( newfluxfull), copy.deepcopy(newfluxfit) slf._fluefull, slf._fluefit = copy.deepcopy(fluefull), copy.deepcopy( fluefit) p0new = [] if slf._argflag['sim']['newstart']: cntr = 0 for pw in range(len(slf._modpass['p0'])): if perror[pw] > 0.0: p0new.append(slf._modpass['p0'][pw] + (Y_covar_fit[cntr, sim]).flatten()[0]) cntr += 1 else: p0new.append(slf._modpass['p0'][pw]) else: p0new = slf._modpass['p0'] #alsave.save_model(slf, p0new, mfit.perror, [(0.0 - 0.0)/3600.0, mfit.fnorm, mfit.dof, mfit.niter, mfit.status], printout=False, extratxt=[slf._argflag['sim']['dirname']+'/',".new"]) #np.savetxt(slf._argflag['run']['modname']+'.covar',mfit.covar) fdict = slf.__dict__.copy() fa = {'x': wavf, 'y': fluf, 'err': errf, 'fdict': fdict} # Calculate the starting chi-squared start_func = myfunct_wrap(slf._modpass['p0'], output=2, **fa) slf._chisq_init = np.sum(((fluf - start_func) / errf)**2) if np.isnan(slf._chisq_init): msgs.error("Initial chi-squared is not a number") if slf._chisq_init == np.Inf: msgs.error("Input chi-squared is Infinite" + msgs.newline() + "Perhaps the error spectrum is zero?") # Fit the realisation # msgs.info("Using {0:d} CPUs".format(slf._argflag['run']['ncpus']),verbose=slf._argflag['out']['verbose']) tstart = time.time() mr = alfit(myfunct_wrap, slf._modpass['p0'], parinfo=parinfo, functkw=fa, verbose=1, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) # mr = alfit(myfunct_wrap, slf._modpass['p0'], parinfo=parinfo, functkw=fa, # verbose=0, modpass=slf._modpass, miniter=slf._argflag['chisq']['miniter'], maxiter=slf._argflag['chisq']['maxiter'], # ftol=slf._argflag['chisq']['ftol'], gtol=slf._argflag['chisq']['gtol'], xtol=slf._argflag['chisq']['xtol'], # ncpus=slf._argflag['run']['ncpus'], fstep=slf._argflag['chisq']['fstep']) tend = time.time() if mr.status <= 0: if mr.status == -20: msgs.info("Random simulation was interrupted", verbose=slf._argflag['out']['verbose']) return else: msgs.error(mr.errmsg) else: msgs.info("Reason for convergence:" + msgs.newline() + alutils.getreason( mr.status, verbose=slf._argflag['out']['verbose']), verbose=slf._argflag['out']['verbose']) if mr.perror is None: msgs.bug("Errors returned from random fit is None", verbose=slf._argflag['out']['verbose']) msgs.error("Cannot continue with the simulations") # Get the results and print them to file outrand = np.append(outrand, np.array([np.array(mr.params)]), axis=0) alsave.save_model( slf, mr.params, mr.perror, [(tend - tstart) / 3600.0, mr.fnorm, mr.dof, mr.niter, mr.status], printout=False, extratxt=[slf._argflag['sim']['dirname'] + '/', ".rand" + ntxt]) # Plot the data (if requested) if slf._argflag['plot']['fits']: model = myfunct_wrap(mr.params, output=3, **fa) alplot.make_plots_all(slf, model=model) fileend = raw_input(msgs.input() + "Press enter to view the fits -") alplot.plot_showall() if slf._argflag['sim']['systematics']: # Calculate the systematics msgs.test( "SYSTEMATIC ERRORS -- Realisation {0:s}/{1:s} began {2:s}". format(str(sim + 1), str(slf._argflag['sim']['random']), time.ctime()), verbose=slf._argflag['out']['verbose']) ms = sim_systematics(slf, p0new, parinfo, ntxt, edgearr) outsyst = np.append(outsyst, np.array([np.array(ms.params)]), axis=0) ntxt = ":0" + nchr + "d}" outname = "{0:s}.{1:s}_{2" + ntxt + "-{3" + ntxt msgs.info("Saving the results from the random simulations", verbose=slf._argflag['out']['verbose']) randname = outname.format( slf._argflag['run']['modname'], 'rand', slf._argflag['sim']['startid'], slf._argflag['sim']['random'] + slf._argflag['sim']['startid']) np.savetxt(randname, outrand) if slf._argflag['sim']['systematics']: msgs.info("Saving the results from the systematics simulations", verbose=slf._argflag['out']['verbose']) systname = outname.format( slf._argflag['run']['modname'], 'syst', slf._argflag['sim']['startid'], slf._argflag['sim']['random'] + slf._argflag['sim']['startid']) np.savetxt(systname, outsyst) return
def save_model(slf, params, errors, info, printout=True, extratxt=["", ""], filename=None, getlines=False, save=True): """ Save the input model into an output script that can be run as input. """ msgs.info("Saving the best-fitting model parameters", verbose=slf._argflag['out']['verbose']) if filename is None: filename = extratxt[0] + slf._argflag['run'][ 'modname'] + '.out' + extratxt[1] prestringA = "#\n# Generated by ALIS on {0:s}\n#\n".format( datetime.datetime.now().strftime("%d/%m/%y at %H:%M:%S")) prestringA += "# Running Time (hrs) = {0:f}\n".format(info[0]) prestringA += "# Initial Chi-Squared = {0:f}\n".format(slf._chisq_init) prestringA += "# Bestfit Chi-Squared = {0:f}\n".format(info[1]) prestringA += "# Degrees-of-Freedom = {0:d}\n".format(info[2]) prestringA += "# Num. of Iterations = {0:d}\n".format(info[3]) prestringA += "# Convergence Reason = {0:s}\n".format( getreason(info[4], verbose=slf._argflag['out']['verbose'])) prestringA += "\n" inputmodl = "#\n" for i in range(len(slf._parlines)): prestringA += slf._parlines[i] inputmodl += "# " + slf._parlines[i] prestringA += "\ndata read\n" inputmodl += "# data read\n" for i in range(len(slf._datlines)): #prestring += slf._datlines[i] inputmodl += "# " + slf._datlines[i] prestringB = "data end\n" inputmodl += "# data end\n" prestringB += "\nmodel read\n" inputmodl += "# model read\n" modcomlin = [] modcomind = [] toutstring = '' for i in range(len(slf._modlines)): if len(slf._modlines[i].strip()) == 0: # Nothing on a line inputmodl += "# " + slf._modlines[i] continue if slf._modlines[i].split()[0] in ["fix", "lim"]: toutstring += slf._modlines[i].replace('\t', ' ') if slf._modlines[i].lstrip()[0] == '#': modcomlin.append(slf._modlines[i].rstrip('\n')) modcomind.append(i) inputmodl += "# " + slf._modlines[i] outstring, errstring, arrstring = print_model( params, slf._modpass, errs=errors, verbose=slf._argflag['out']['verbose'], funcarray=slf._funcarray) cvstring, cvestring, cvastring = arrstring[0], arrstring[1], arrstring[2] shstring, shestring, shastring = arrstring[3], arrstring[4], arrstring[5] if printout and slf._argflag['out']['verbose'] != -1: print "\n####################################################" print outstring print errstring print "#" + "\n#".join( cvstring.replace("Convolution", "Convolution Models:").split("\n")) print cvestring.replace("#Convolution\n", "") print "#" + "\n#".join( shstring.replace("Shift", "Shift Models:").split("\n")) print shestring.replace("#Shift\n", "") + "\n" print "####################################################\n" # Reinsert the comments at the original locations outstrspl = (toutstring + outstring).split('\n') for i in range(len(modcomlin)): outstrspl.insert(modcomind[i], modcomlin[i]) outstring = '\n'.join(outstrspl) # Include an end tag for the model outstring += "model end\n" inputmodl += "# model end\n#\n\n" # Include the model links if len(slf._lnklines) != 0: outstring += "\nlink read\n" for i in range(len(slf._lnklines)): outstring += slf._lnklines[i] outstring += "link end\n" # Update datlines for the newly derived instrument resolution cnum = 0 snum = 0 dstrarr = ["" for all in slf._datlines] for sp in range(len(slf._specid)): for i in range(len(slf._datlines)): if slf._datlines[i].lstrip() == "": continue # This line is needed for OneFits. if slf._datlines[i].lstrip()[0] == "#": dstrarr[i] += slf._datlines[i] datspl = slf._datlines[i].split() spmatch = False for j in range(1, len(datspl)): dspl = datspl[j].split("=") if dspl[0] == "specid": if dspl[1] == slf._specid[sp]: spmatch = True break if not spmatch: continue gotres = False gotshf = False for j in range(1, len(datspl)): dspl = datspl[j].split("=") if dspl[0] == "resolution": cspl = cvastring.split("\n")[cnum].split() cpars = ",".join(cspl[1:]) datspl[j] = "resolution={0:s}({1:s})".format( cspl[0], cpars) gotres = True elif dspl[0] == "shift": sspl = shastring.split("\n")[snum].split() spars = ",".join(sspl[1:]) datspl[j] = "shift={0:s}({1:s})".format(sspl[0], spars) gotshf = True cnum += 1 snum += 1 # if not gotres: # dstrarr[i] += slf._datlines[i] # else: # dstrarr[i] += " " + " ".join(datspl) + "\n" dstrarr[i] += " " + " ".join(datspl) + "\n" datstring = "".join(dstrarr) # Save the output if save: if slf._argflag['out']['overwrite']: ans = 'y' else: ans = '' if os.path.exists(filename): while ans != 'y' and ans != 'n' and ans != 'r': msgs.warn("File %s exists!" % (filename), verbose=slf._argflag['out']['verbose']) ans = raw_input(msgs.input() + "Overwrite? (y/n) or rename? (r) - ") if ans == 'r': fileend = raw_input(msgs.input() + "Enter new filename - ") filename = fileend if os.path.exists(filename): ans = '' if ans != 'n': infile = open(filename, "w") infile.write(prestringA) infile.write(datstring) infile.write(prestringB) infile.write(outstring) infile.write("\n" + errstring + "\n") infile.write("#" + "\n#".join( cvstring.replace("Convolution", "Convolution Models:").split( "\n"))) infile.write("\n" + cvestring.replace("#Convolution\n", "") + "\n") infile.write("#" + "\n#".join( shstring.replace("Shift", "Shift Models:").split("\n"))) infile.write("\n" + shestring.replace("#Shift\n", "") + "\n") infile.write( "\n###################################################") infile.write( "\n# #") infile.write( "\n# HERE IS A COPY OF THE INPUT MODEL #") infile.write( "\n# #") infile.write( "\n###################################################\n") infile.write(inputmodl) infile.close() msgs.info("Saved output file successfully:" + msgs.newline() + filename, verbose=slf._argflag['out']['verbose']) if getlines: sendstr = prestringA + datstring + prestringB + outstring + "\n" + errstring + "\n" sendstr += "#" + "\n#".join( cvstring.replace("Convolution", "Convolution Models:").split("\n")) sendstr += "\n" + cvestring.replace("#Convolution\n", "") + "\n" sendstr += "#" + "\n#".join( shstring.replace("Shift", "Shift Models:").split("\n")) sendstr += "\n" + shestring.replace("#Shift\n", "") + "\n" return sendstr
def save_model(slf,params,errors,info,printout=True,extratxt=["",""],filename=None,getlines=False,save=True): """ Save the input model into an output script that can be run as input. """ msgs.info("Saving the best-fitting model parameters", verbose=slf._argflag['out']['verbose']) if filename is None: filename = extratxt[0]+slf._argflag['run']['modname']+'.out'+extratxt[1] prestringA = "#\n# Generated by ALIS on {0:s}\n#\n".format(datetime.datetime.now().strftime("%d/%m/%y at %H:%M:%S")) prestringA += "# Running Time (hrs) = {0:f}\n".format(info[0]) prestringA += "# Initial Chi-Squared = {0:f}\n".format(slf._chisq_init) prestringA += "# Bestfit Chi-Squared = {0:f}\n".format(info[1]) prestringA += "# Degrees-of-Freedom = {0:d}\n".format(info[2]) prestringA += "# Num. of Iterations = {0:d}\n".format(info[3]) prestringA += "# Convergence Reason = {0:s}\n".format(getreason(info[4],verbose=slf._argflag['out']['verbose'])) prestringA += "\n" inputmodl = "#\n" for i in range(len(slf._parlines)): prestringA += slf._parlines[i] inputmodl += "# "+slf._parlines[i] prestringA +="\ndata read\n" inputmodl += "# data read\n" for i in range(len(slf._datlines)): #prestring += slf._datlines[i] inputmodl += "# "+slf._datlines[i] prestringB ="data end\n" inputmodl += "# data end\n" prestringB +="\nmodel read\n" inputmodl += "# model read\n" modcomlin=[] modcomind=[] toutstring='' for i in range(len(slf._modlines)): if len(slf._modlines[i].strip()) == 0: # Nothing on a line inputmodl += "# "+slf._modlines[i] continue if slf._modlines[i].split()[0] in ["fix", "lim"]: toutstring += slf._modlines[i].replace('\t',' ') if slf._modlines[i].lstrip()[0] == '#': modcomlin.append(slf._modlines[i].rstrip('\n')) modcomind.append(i) inputmodl += "# "+slf._modlines[i] outstring, errstring, arrstring = print_model(params,slf._modpass,errs=errors,verbose=slf._argflag['out']['verbose'],funcarray=slf._funcarray) cvstring, cvestring, cvastring = arrstring[0], arrstring[1], arrstring[2] shstring, shestring, shastring = arrstring[3], arrstring[4], arrstring[5] if printout and slf._argflag['out']['verbose'] != -1: print "\n####################################################" print outstring print errstring print "#"+"\n#".join(cvstring.replace("Convolution","Convolution Models:").split("\n")) print cvestring.replace("#Convolution\n","") print "#"+"\n#".join(shstring.replace("Shift","Shift Models:").split("\n")) print shestring.replace("#Shift\n","")+"\n" print "####################################################\n" # Reinsert the comments at the original locations outstrspl = (toutstring+outstring).split('\n') for i in range(len(modcomlin)): outstrspl.insert(modcomind[i],modcomlin[i]) outstring = '\n'.join(outstrspl) # Include an end tag for the model outstring += "model end\n" inputmodl += "# model end\n#\n\n" # Include the model links if len(slf._lnklines) != 0: outstring += "\nlink read\n" for i in range(len(slf._lnklines)): outstring += slf._lnklines[i] outstring += "link end\n" # Update datlines for the newly derived instrument resolution cnum=0 snum=0 dstrarr = ["" for all in slf._datlines] for sp in range(len(slf._specid)): for i in range(len(slf._datlines)): if slf._datlines[i].lstrip() == "": continue # This line is needed for OneFits. if slf._datlines[i].lstrip()[0] == "#": dstrarr[i] += slf._datlines[i] datspl = slf._datlines[i].split() spmatch = False for j in range(1,len(datspl)): dspl = datspl[j].split("=") if dspl[0] == "specid": if dspl[1] == slf._specid[sp]: spmatch = True break if not spmatch: continue gotres=False gotshf=False for j in range(1,len(datspl)): dspl = datspl[j].split("=") if dspl[0] == "resolution": cspl = cvastring.split("\n")[cnum].split() cpars = ",".join(cspl[1:]) datspl[j] = "resolution={0:s}({1:s})".format(cspl[0],cpars) gotres = True elif dspl[0] == "shift": sspl = shastring.split("\n")[snum].split() spars = ",".join(sspl[1:]) datspl[j] = "shift={0:s}({1:s})".format(sspl[0],spars) gotshf = True cnum += 1 snum += 1 # if not gotres: # dstrarr[i] += slf._datlines[i] # else: # dstrarr[i] += " " + " ".join(datspl) + "\n" dstrarr[i] += " " + " ".join(datspl) + "\n" datstring = "".join(dstrarr) # Save the output if save: if slf._argflag['out']['overwrite']: ans='y' else: ans='' if os.path.exists(filename): while ans != 'y' and ans != 'n' and ans !='r': msgs.warn("File %s exists!" % (filename), verbose=slf._argflag['out']['verbose']) ans = raw_input(msgs.input()+"Overwrite? (y/n) or rename? (r) - ") if ans == 'r': fileend=raw_input(msgs.input()+"Enter new filename - ") filename = fileend if os.path.exists(filename): ans = '' if ans != 'n': infile = open(filename,"w") infile.write(prestringA) infile.write(datstring) infile.write(prestringB) infile.write(outstring) infile.write("\n"+errstring+"\n") infile.write("#"+"\n#".join(cvstring.replace("Convolution","Convolution Models:").split("\n"))) infile.write("\n"+cvestring.replace("#Convolution\n","")+"\n") infile.write("#"+"\n#".join(shstring.replace("Shift","Shift Models:").split("\n"))) infile.write("\n"+shestring.replace("#Shift\n","")+"\n") infile.write("\n###################################################") infile.write("\n# #") infile.write("\n# HERE IS A COPY OF THE INPUT MODEL #") infile.write("\n# #") infile.write("\n###################################################\n") infile.write(inputmodl) infile.close() msgs.info("Saved output file successfully:"+msgs.newline()+filename, verbose=slf._argflag['out']['verbose']) if getlines: sendstr = prestringA + datstring + prestringB + outstring + "\n"+errstring+"\n" sendstr += "#"+"\n#".join(cvstring.replace("Convolution","Convolution Models:").split("\n")) sendstr += "\n"+cvestring.replace("#Convolution\n","")+"\n" sendstr += "#"+"\n#".join(shstring.replace("Shift","Shift Models:").split("\n")) sendstr += "\n"+shestring.replace("#Shift\n","")+"\n" return sendstr
def save_convtest(slf, diff, thresh, info, printout=True, extratxt=["", ""]): """ Save the details of what parameters have converged. """ msgs.info("Saving the best-fitting model parameters", verbose=slf._argflag["out"]["verbose"]) filename = extratxt[0] + slf._argflag["run"]["modname"] + ".conv" + extratxt[1] prestring = "#\n# Generated by ALIS on {0:s}\n#\n".format(datetime.datetime.now().strftime("%d/%m/%y at %H:%M:%S")) prestring += "# Running Time (hrs) = {0:f}\n".format(info[0]) prestring += "# Initial Chi-Squared = {0:f}\n".format(slf._chisq_init) prestring += "# Bestfit Chi-Squared = {0:f}\n".format(info[1]) prestring += "# Degrees-of-Freedom = {0:d}\n".format(info[2]) prestring += "# Num. of Iterations = {0:d}\n".format(info[3]) prestring += "# Convergence Reason = {0:s}\n".format(getreason(info[4], verbose=slf._argflag["out"]["verbose"])) prestring += "\n" inputmodl = "#\n" for i in range(len(slf._parlines)): prestring += slf._parlines[i] inputmodl += "# " + slf._parlines[i] prestring += "\ndata read\n" inputmodl += "# data read\n" msgs.bug("INCLUDE THE BEST-FIT FWHM HERE --- PLACE THE ERROR BELOW", verbose=slf._argflag["out"]["verbose"]) for i in range(len(slf._datlines)): prestring += slf._datlines[i] inputmodl += "# " + slf._datlines[i] prestring += "data end\n" inputmodl += "# data end\n" prestring += "\nmodel read\n" inputmodl += "# model read\n" modcomlin = [] modcomind = [] tconvstring = "" for i in range(len(slf._modlines)): if slf._modlines[i].split()[0] in ["fix", "lim"]: tconvstring += slf._modlines[i].replace("\t", " ") if slf._modlines[i].lstrip()[0] == "#": modcomlin.append(slf._modlines[i].rstrip("\n")) modcomind.append(i) inputmodl += "# " + slf._modlines[i] convstring = print_model( diff, slf._modpass, thresh, verbose=slf._argflag["out"]["verbose"], funcarray=slf._funcarray ) # if printout and slf._argflag['out']['verbose'] != -1: # print "\n####################################################" # print convstring # print "####################################################\n" # Reinsert the comments at the original locations cnvstrspl = (tconvstring + convstring).split("\n") for i in range(len(modcomlin)): cnvstrspl.insert(modcomind[i], modcomlin[i]) convstring = "\n".join(cnvstrspl) # Include an end tag for the model convstring += "model end\n" inputmodl += "# model end\n#\n\n" if slf._argflag["out"]["overwrite"]: ans = "y" else: ans = "" if os.path.exists(filename): while ans != "y" and ans != "n" and ans != "r": msgs.warn("File %s exists!" % (filename), verbose=slf._argflag["out"]["verbose"]) ans = raw_input(msgs.input() + "Overwrite? (y/n) or rename? (r) - ") if ans == "r": fileend = raw_input(msgs.input() + "Enter new filename - ") filename = fileend if os.path.exists(filename): ans = "" if ans != "n": infile = open(filename, "w") infile.write(prestring) infile.write(convstring + "\n") infile.write("\n###################################################") infile.write("\n# #") infile.write("\n# HERE IS A COPY OF THE INPUT MODEL #") infile.write("\n# #") infile.write("\n###################################################\n") infile.write(inputmodl) infile.close() msgs.info("Saved output file successfully:" + msgs.newline() + filename, verbose=slf._argflag["out"]["verbose"])