def mock_update_run(custom_command=""): """ Tentative covmat update run Not reachable yet by any option. """ from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() # store the command_line space if not custom_command: custom_command = " ".join(sys.argv[1:]) # Do a first run mpi_run(custom_command) # Compute the covariance matrix if rank == 0: info_command = from_run_to_info(custom_command) initialise(info_command) # Make sure that the covariance matrix used is the one just computed with_covmat = add_covariance_matrix(custom_command) mpi_run(with_covmat) return
def safe_initialisation(custom_command="", comm=None, nprocs=1): """ Wrapper around the init function to handle errors KeyWord Arguments ----------------- custom_command : str testing purposes comm : MPI.Intracomm object that helps communicating between the processes nprocs : int number of processes """ try: cosmo, data, command_line, success = initialise(custom_command) except io_mp.ConfigurationError as message: if comm: for index in range(1, nprocs): comm.send('failed', dest=index, tag=1) print str(message) raise io_mp.ConfigurationError( "The initialisation was not successful, resulting in a " "potentially half created `log.param`. Please see the " "above error message. If you run the exact same command, it" " will not work. You should solve the problem, and try again.") except KeyError: if comm: for index in range(1, nprocs): comm.send('failed', dest=index, tag=1) raise io_mp.ConfigurationError( "You are running in a folder that was created following " "a non-successful initialisation (wrong parameter name, " "wrong likelihood, etc...). If you have solved the issue, you " "should remove completely the output folder, and try again.") return cosmo, data, command_line, success
def run(custom_command=''): """ Main call of the function It recovers the initialised instances of cosmo Class, :class:`Data` and the NameSpace containing the command line arguments, feeding into the sampler. .. note:: A possible parallelization would take place here. Parameters ---------- custom_command: str allows for testing the code """ # Initialisation routine cosmo, data, command_line, success = initialise(custom_command) # If success is False, it means either that the initialisation was not # successful, or that it was simply an analysis call. The run should stop if not success: return # Once that the initialisation phase is done, one can import the sampler import sampler # Generic sampler call sampler.run(cosmo, data, command_line) return
def normaliser(dx,dt,tau): lats=np.arange(0, 1E7+dx, dx) lons=np.arange(0, 1E7+dx, dx) matrix = np.loadtxt("TM_dt{0}_dx{1}_tau{2}_matrix_count.dat".format(dt,dx,tau),unpack=True) normalized = normalize(initialise(lons,lats,matrix), norm='l1', axis=1) normalized = lil_matrix(normalized) file = open("./TM_dt{0}_dx{1}_tau{2}_matrix_rownorm.dat".format(dt,dx,tau),"w") file.write("{:}".format(normalized)) file.close()
def normaliser(dpsi,dtheta,dt,tau): psigrid=np.arange(0, 320000+dpsi, dpsi) tgrid=np.arange(0, 360+dtheta, dtheta) matrix = np.loadtxt("PTM_dt{0}_dpsi{1}_dtheta{2}_tau{3}_matrix_count.dat".format(dt,dpsi,dtheta,tau),unpack=True) normalized = normalize(initialise(psigrid,tgrid,matrix), norm='l1', axis=1) normalized = lil_matrix(normalized) file = open("./PTM_dt{0}_dpsi{1}_dtheta{2}_tau{3}_matrix_rownorm.dat".format(dt,dpsi,dtheta,tau),"w") file.write("{:}".format(normalized)) file.close()
def read(self): xdg = False root = os.getenv("XDG_CONFIG_HOME") if root != None: xdg = True if xdg: config_path = root + "/steam_launch_options_manager.config" status = self.read_config(config_path) if status: return root = os.getenv("HOME") + "/.config" config_path = root + "/steam_launch_options_manager.config" status = self.read_config(config_path) if status: return print( "No config for steam launch options manager found. Starting initialization. \n" ) initialise.initialise().initialise()
def normaliser(dadt,dx,dt,tau): lats=np.arange(-90, 90+dx, dx) lons=np.arange(0, 360+dx, dx) adt_range=np.arange(-2,2.2+dadt,dadt) if dx==0.5: dx="half" matrix = np.loadtxt("TM_dx{1}_dt{0}_tau{2}_dadtp2_count.dat".format(dt,dx,tau),unpack=True) normalized = normalize(initialise(lons,lats,adt_range,matrix), norm='l1', axis=1) normalized = lil_matrix(normalized) file = open("./TM_dx{0}_dt{1}_tau{2}_dadtp2_rownorm.dat".format(dx,dt,tau),"w") file.write("{:}".format(normalized)) file.close()
def mpi_run(custom_command=""): """ Launch a simple MPI run, with no communication of covariance matrix It simply allows the first process to create the folder - so that the log.param is properly written. A signal is then send to the other processes, that contains the chain number of the parent run. In order to be sure to have different chain numbers, it adds the rank of the process and the initial job number - this should avoid conflict, but can be subject of future improvements """ from mpi4py import MPI comm = MPI.COMM_WORLD nprocs = comm.Get_size() rank = comm.Get_rank() # this will be the master cpu. This guy will create - or append - a folder, # being sure to be the first to do so. if rank == 0: # First initialisation cosmo, data, command_line, success = safe_initialisation( custom_command, comm, nprocs) regexp = re.match(".*__(\w*).txt", data.out_name) suffix = regexp.groups()[0] # Send an "OK" signal to all the other processes, actually giving the # suffix of this master chain. All the other will add 1 to this number for index in range(1, nprocs): comm.send(suffix, dest=index, tag=1) else: # If the rank is not 0, it is a slave process. It waits to receive the # "OK" message, which is immediatly discarded. suffix = comm.recv(source=0, tag=1) # If a failed message was passed, exit the process if suffix == 'failed': return # Concatenate the rank to the suffix, and not the opposite, this should # avoid any conflicting name if not custom_command: custom_command = " ".join(sys.argv[1:]) custom_command += " --chain-number %s" % str(rank)+suffix cosmo, data, command_line, success = initialise(custom_command) import sampler sampler.run(cosmo, data, command_line) return
def main(): # read in config for steam launch options manger config = config_reader.config_reader() config.read() # read in steam config p = steam_config_utils.config_getter() data = p.open_file(config.steam_path) # create backup p.backup_config(data, initialise.initialise().get_backup_path(), "localconfig.vdf.backup"+str(int(time.time()))) # modify steam backup modifer = modify_config.modifier(config.rules) modifer.modify(data) print("Saving modified file") p.save_file(data, config.steam_path) pass
def safe_initialisation(custom_command="", comm=None, nprocs=1): """ Wrapper around the init function to handle errors KeyWord Arguments ----------------- custom_command : str testing purposes comm : MPI.Intracomm object that helps communicating between the processes nprocs : int number of processes """ try: cosmo1, cosmo2, data, command_line, success = initialise( custom_command) except io_mp.ConfigurationError as message: if comm: for index in range(1, nprocs): comm.send('failed', dest=index, tag=1) print(str(message)) raise io_mp.ConfigurationError( "The initialisation was not successful, resulting in a " "potentially half created `log.param`. Please see the " "above error message. If you run the exact same command, it" " will not work. You should solve the problem, and try again.") except KeyError as e: if comm: for index in range(1, nprocs): comm.send('failed', dest=index, tag=1) raise io_mp.ConfigurationError( "You are running in a folder that was created following " "a non-successful initialisation (wrong parameter name, " "wrong likelihood, etc...). If you have solved the issue, you " "should remove completely the output folder, and try again." + " Alternatively, there could be a problem with " + e.message) return cosmo1, cosmo2, data, command_line, success
RESUMES = STATIC + '/resumes' BEDRIJVENDAGBOEK = ROOT + '/bedrijvendagboek' session_opts = { 'session.type': 'file', 'session.data_dir': './session/', 'session.cookie_expires': 1800, 'session.auto': True, } app = beaker.middleware.SessionMiddleware(bottle.app(), session_opts) message_flash = FlashPlugin(key='messages', secret=secret_key) Jinja2Template.defaults["get_flashed_messages"] = message_flash.get_flashed_messages Jinja2Template.settings["extensions"] = ["jinja2.ext.with_"] initialise.initialise() @bottle.hook('before_request') def setup_session(): request.session = bottle.request.environ.get('beaker.session') @bottle.get('/favicon.ico') def get_favicon(): return None @bottle.route('/') def index(): """returns static template index"""
""" # calling the parent method initialization dict.__init__(self) self['initial'] = array[0:4] self['scale'] = array[4] self['role'] = array[-1] self['tex_name'] = io_mp.get_tex_name(key) if array[3] == 0: self['status'] = 'fixed' self['current'] = array[0] else: self['status'] = 'varying' self['prior'] = prior.Prior(array) class Container(object): """Dummy class to act as a namespace for data""" pass if __name__ == "__main__": import doctest import shutil from initialise import initialise folder = os.path.join('tests', 'doc') cosmo, data, command_line, _ = initialise('-o %s -p test.param' % folder) doctest.testmod(extraglobs={'data': data}) shutil.rmtree(folder)
def mpi_run(custom_command=""): """ Launch a simple MPI run, with no communication of covariance matrix It simply allows the first process to create the folder - so that the log.param is properly written. A signal is then send to the other processes, that contains the chain number of the parent run. In order to be sure to have different chain numbers, it adds the rank of the process and the initial job number - this should avoid conflict, but can be subject of future improvements """ from mpi4py import MPI comm = MPI.COMM_WORLD nprocs = comm.Get_size() rank = comm.Get_rank() # this will be the master cpu. This guy will create - or append - a folder, # being sure to be the first to do so. if rank == 0: # First initialisation cosmo, data, command_line, success = safe_initialisation( custom_command, comm, nprocs) # Check that the run asked is compatible with mpirun and prepare. if command_line.subparser_name == 'info': warnings.warn( "Analyzing the chains is not supported in mpirun" " so this will run on one core only.") status = 'failed' elif command_line.method == "MH": regexp = re.match(".*__(\w*).txt", data.out_name) suffix = regexp.groups()[0] status = suffix elif command_line.method == "NS": status = 1 else: warnings.warn( "The method '%s' is not supported"%(command_line.method) + " in mpirun so this will run on one core only.") status = 'failed' # Send an "OK" signal to all the other processes, actually giving the # suffix of this master chain. All the other will add 1 to this number for index in range(1, nprocs): comm.send(status, dest=index, tag=1) else: # If the rank is not 0, it is a slave process. It waits to receive the # "OK" message, which is immediatly discarded. status = comm.recv(source=0, tag=1) # If a failed message was passed, exit the process if status == 'failed': success = False else: # Concatenate the rank to the suffix, and not the opposite, this # should avoid any conflicting name if not custom_command: custom_command = " ".join(sys.argv[1:]) suffix = status custom_command += " --chain-number %s" % str(int(suffix)+rank) cosmo, data, command_line, success = initialise(custom_command) if success: import sampler sampler.run(cosmo, data, command_line) success = comm.gather(success, root=0) return
# calling the parent method initialization dict.__init__(self) self["initial"] = array[0:4] self["scale"] = array[4] self["role"] = array[-1] self["tex_name"] = io_mp.get_tex_name(key) if array[3] == 0: self["status"] = "fixed" self["current"] = array[0] else: self["status"] = "varying" self["prior"] = prior.Prior(array) class Container(object): """Dummy class to act as a namespace for data""" pass if __name__ == "__main__": import doctest import shutil from initialise import initialise folder = os.path.join("tests", "doc") cosmo, data, command_line, _ = initialise("-o %s -p test.param" % folder) doctest.testmod(extraglobs={"data": data}) shutil.rmtree(folder)
import numpy as np import constant import initialise import interpolate import preprocess import save if __name__ == '__main__': algorithm_start_time = time() print('Initialize the algorithm...') start_time = time() action, training_files, testing_files, squares, features = initialise.initialise( ) squares_training_data = preprocess.load_squares(training_files, squares) squares_testing_data = preprocess.load_squares(testing_files, squares) print('Done to initialze the algorithm. Time: ', round(time() - start_time, 3), ' sec') for square in squares: print('*** SQUARE ' + str(square) + ' ***') if (square not in squares_training_data) or (square not in squares_testing_data): print('No data for square (' + str(square) + '), skip.')
def init(): """Initialise tracking tools and database.""" initialise.initialise()
# path for all necessary input paths = {} # cosmological code path (CLASS) paths['cosmo'] = '/net/eemmeer/data1/ssli/class_public' # father path for all the input and output paths['data'] = '/disks/shear15/ssli/CosmicShear' # parameter/configure file path paths[ 'param'] = '/net/raam/data1/surfdrive_ssli/Projects/6CosmicShear_RB/CosmicShearRB/Cosmo/cosmic_shear_signal/input' # +++++++++++++++++++ Running scripts Start = time.time() # Initialisation # class: data and cosmo created cosmo, data = initialise.initialise(paths) for i in range(len(name_param_files)): # parameter file (with cosmological and nuisance parameters) name_param_file = name_param_files[i] data.read_file(name_param_file, 'data', field='', separate=False) # output folder data.conf['out_folder'] = out_folder data.conf['out_suffix'] = out_suffix_s[i] # # ++++++++++++++++++++++++++++++++++++++++++ whole # # name of configure files # name_conf_file = 'kv450_cf.conf' # # data filled with input files
# # Number of bins # nzbins = len(data_whole.const['z_bins_min']) # # Number of correlation # nzcorrs = int(nzbins * (nzbins + 1) / 2) # # # chi2_whole, dof_whole = Chi2Test.Chi2SingleFunc(nzbins, nzcorrs, theta_bins_whole, mask_whole, data_whole, xi_obs_whole, xi_theo_whole) # # print("chi2_whole", chi2_whole) # # print("dof_whole", dof_whole) # ++++++++++++++++++++++++++++++++++++++++++ red # name of parameter/configure files name_conf_file = 'kv450_cf_red.conf' # Initialisation # class: data and cosmo created cosmo, data_red = initialise.initialise(paths) # data filled with input files # parameter file (with cosmological and nuisance parameters) data_red.read_file(name_param_file, 'data', field='', separate=False) # configure file (with configure and hardly changed setting parameters) data_red.read_file(name_conf_file, 'data', field='', separate=False) # cosmic shear signal calculation xi_obs_red, xi_theo_red, theta_bins_red, mask_red = CosmicShear.CSsignalFunc( data_red, cosmo, save_theory_vector=False) # xi_obs_red, xi_theo_red, theta_bins_red, mask_red = CosmicShear.CSsignalFunc(data_red, cosmo, True) # chi2 # Number of bins nzbins = len(data_red.const['z_bins_min']) # Number of correlation