def mock_update_run(custom_command=""):
"""
Tentative covmat update run
Not reachable yet by any option.
"""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
# store the command_line space
if not custom_command:
custom_command = " ".join(sys.argv[1:])
# Do a first run
mpi_run(custom_command)
# Compute the covariance matrix
if rank == 0:
info_command = from_run_to_info(custom_command)
initialise(info_command)
# Make sure that the covariance matrix used is the one just computed
with_covmat = add_covariance_matrix(custom_command)
mpi_run(with_covmat)
return
def mock_update_run(custom_command=""):
"""
Tentative covmat update run
Not reachable yet by any option.
"""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
# store the command_line space
if not custom_command:
custom_command = " ".join(sys.argv[1:])
# Do a first run
mpi_run(custom_command)
# Compute the covariance matrix
if rank == 0:
info_command = from_run_to_info(custom_command)
initialise(info_command)
# Make sure that the covariance matrix used is the one just computed
with_covmat = add_covariance_matrix(custom_command)
mpi_run(with_covmat)
return
def safe_initialisation(custom_command="", comm=None, nprocs=1):
"""
Wrapper around the init function to handle errors
KeyWord Arguments
-----------------
custom_command : str
testing purposes
comm : MPI.Intracomm
object that helps communicating between the processes
nprocs : int
number of processes
"""
try:
cosmo, data, command_line, success = initialise(custom_command)
except io_mp.ConfigurationError as message:
if comm:
for index in range(1, nprocs):
comm.send('failed', dest=index, tag=1)
print str(message)
raise io_mp.ConfigurationError(
"The initialisation was not successful, resulting in a "
"potentially half created `log.param`. Please see the "
"above error message. If you run the exact same command, it"
" will not work. You should solve the problem, and try again.")
except KeyError:
if comm:
for index in range(1, nprocs):
comm.send('failed', dest=index, tag=1)
raise io_mp.ConfigurationError(
"You are running in a folder that was created following "
"a non-successful initialisation (wrong parameter name, "
"wrong likelihood, etc...). If you have solved the issue, you "
"should remove completely the output folder, and try again.")
return cosmo, data, command_line, success
def run(custom_command=''):
"""
Main call of the function
It recovers the initialised instances of cosmo Class, :class:`Data` and the
NameSpace containing the command line arguments, feeding into the sampler.
.. note::
A possible parallelization would take place here.
Parameters
----------
custom_command: str
allows for testing the code
"""
# Initialisation routine
cosmo, data, command_line, success = initialise(custom_command)
# If success is False, it means either that the initialisation was not
# successful, or that it was simply an analysis call. The run should stop
if not success:
return
# Once that the initialisation phase is done, one can import the sampler
import sampler
# Generic sampler call
sampler.run(cosmo, data, command_line)
return
def normaliser(dx,dt,tau):
lats=np.arange(0, 1E7+dx, dx)
lons=np.arange(0, 1E7+dx, dx)
matrix = np.loadtxt("TM_dt{0}_dx{1}_tau{2}_matrix_count.dat".format(dt,dx,tau),unpack=True)
normalized = normalize(initialise(lons,lats,matrix), norm='l1', axis=1)
normalized = lil_matrix(normalized)
file = open("./TM_dt{0}_dx{1}_tau{2}_matrix_rownorm.dat".format(dt,dx,tau),"w")
file.write("{:}".format(normalized))
file.close()
def normaliser(dpsi,dtheta,dt,tau):
psigrid=np.arange(0, 320000+dpsi, dpsi)
tgrid=np.arange(0, 360+dtheta, dtheta)
matrix = np.loadtxt("PTM_dt{0}_dpsi{1}_dtheta{2}_tau{3}_matrix_count.dat".format(dt,dpsi,dtheta,tau),unpack=True)
normalized = normalize(initialise(psigrid,tgrid,matrix), norm='l1', axis=1)
normalized = lil_matrix(normalized)
file = open("./PTM_dt{0}_dpsi{1}_dtheta{2}_tau{3}_matrix_rownorm.dat".format(dt,dpsi,dtheta,tau),"w")
file.write("{:}".format(normalized))
file.close()
def read(self):
xdg = False
root = os.getenv("XDG_CONFIG_HOME")
if root != None:
xdg = True
if xdg:
config_path = root + "/steam_launch_options_manager.config"
status = self.read_config(config_path)
if status:
return
root = os.getenv("HOME") + "/.config"
config_path = root + "/steam_launch_options_manager.config"
status = self.read_config(config_path)
if status:
return
print(
"No config for steam launch options manager found. Starting initialization. \n"
)
initialise.initialise().initialise()
def normaliser(dadt,dx,dt,tau):
lats=np.arange(-90, 90+dx, dx)
lons=np.arange(0, 360+dx, dx)
adt_range=np.arange(-2,2.2+dadt,dadt)
if dx==0.5:
dx="half"
matrix = np.loadtxt("TM_dx{1}_dt{0}_tau{2}_dadtp2_count.dat".format(dt,dx,tau),unpack=True)
normalized = normalize(initialise(lons,lats,adt_range,matrix), norm='l1', axis=1)
normalized = lil_matrix(normalized)
file = open("./TM_dx{0}_dt{1}_tau{2}_dadtp2_rownorm.dat".format(dx,dt,tau),"w")
file.write("{:}".format(normalized))
file.close()
def mpi_run(custom_command=""):
"""
Launch a simple MPI run, with no communication of covariance matrix
It simply allows the first process to create the folder - so that the
log.param is properly written. A signal is then send to the other
processes, that contains the chain number of the parent run.
In order to be sure to have different chain numbers, it adds the rank of
the process and the initial job number - this should avoid conflict, but
can be subject of future improvements
"""
from mpi4py import MPI
comm = MPI.COMM_WORLD
nprocs = comm.Get_size()
rank = comm.Get_rank()
# this will be the master cpu. This guy will create - or append - a folder,
# being sure to be the first to do so.
if rank == 0:
# First initialisation
cosmo, data, command_line, success = safe_initialisation(
custom_command, comm, nprocs)
regexp = re.match(".*__(\w*).txt", data.out_name)
suffix = regexp.groups()[0]
# Send an "OK" signal to all the other processes, actually giving the
# suffix of this master chain. All the other will add 1 to this number
for index in range(1, nprocs):
comm.send(suffix, dest=index, tag=1)
else:
# If the rank is not 0, it is a slave process. It waits to receive the
# "OK" message, which is immediatly discarded.
suffix = comm.recv(source=0, tag=1)
# If a failed message was passed, exit the process
if suffix == 'failed':
return
# Concatenate the rank to the suffix, and not the opposite, this should
# avoid any conflicting name
if not custom_command:
custom_command = " ".join(sys.argv[1:])
custom_command += " --chain-number %s" % str(rank)+suffix
cosmo, data, command_line, success = initialise(custom_command)
import sampler
sampler.run(cosmo, data, command_line)
return
def main():
# read in config for steam launch options manger
config = config_reader.config_reader()
config.read()
# read in steam config
p = steam_config_utils.config_getter()
data = p.open_file(config.steam_path)
# create backup
p.backup_config(data, initialise.initialise().get_backup_path(),
"localconfig.vdf.backup"+str(int(time.time())))
# modify steam backup
modifer = modify_config.modifier(config.rules)
modifer.modify(data)
print("Saving modified file")
p.save_file(data, config.steam_path)
pass
def safe_initialisation(custom_command="", comm=None, nprocs=1):
"""
Wrapper around the init function to handle errors
KeyWord Arguments
-----------------
custom_command : str
testing purposes
comm : MPI.Intracomm
object that helps communicating between the processes
nprocs : int
number of processes
"""
try:
cosmo1, cosmo2, data, command_line, success = initialise(
custom_command)
except io_mp.ConfigurationError as message:
if comm:
for index in range(1, nprocs):
comm.send('failed', dest=index, tag=1)
print(str(message))
raise io_mp.ConfigurationError(
"The initialisation was not successful, resulting in a "
"potentially half created `log.param`. Please see the "
"above error message. If you run the exact same command, it"
" will not work. You should solve the problem, and try again.")
except KeyError as e:
if comm:
for index in range(1, nprocs):
comm.send('failed', dest=index, tag=1)
raise io_mp.ConfigurationError(
"You are running in a folder that was created following "
"a non-successful initialisation (wrong parameter name, "
"wrong likelihood, etc...). If you have solved the issue, you "
"should remove completely the output folder, and try again." +
" Alternatively, there could be a problem with " + e.message)
return cosmo1, cosmo2, data, command_line, success
RESUMES = STATIC + '/resumes'
BEDRIJVENDAGBOEK = ROOT + '/bedrijvendagboek'
session_opts = {
'session.type': 'file',
'session.data_dir': './session/',
'session.cookie_expires': 1800,
'session.auto': True,
}
app = beaker.middleware.SessionMiddleware(bottle.app(), session_opts)
message_flash = FlashPlugin(key='messages', secret=secret_key)
Jinja2Template.defaults["get_flashed_messages"] = message_flash.get_flashed_messages
Jinja2Template.settings["extensions"] = ["jinja2.ext.with_"]
initialise.initialise()
@bottle.hook('before_request')
def setup_session():
request.session = bottle.request.environ.get('beaker.session')
@bottle.get('/favicon.ico')
def get_favicon():
return None
@bottle.route('/')
def index():
"""returns static template index"""
"""
# calling the parent method initialization
dict.__init__(self)
self['initial'] = array[0:4]
self['scale'] = array[4]
self['role'] = array[-1]
self['tex_name'] = io_mp.get_tex_name(key)
if array[3] == 0:
self['status'] = 'fixed'
self['current'] = array[0]
else:
self['status'] = 'varying'
self['prior'] = prior.Prior(array)
class Container(object):
"""Dummy class to act as a namespace for data"""
pass
if __name__ == "__main__":
import doctest
import shutil
from initialise import initialise
folder = os.path.join('tests', 'doc')
cosmo, data, command_line, _ = initialise('-o %s -p test.param' % folder)
doctest.testmod(extraglobs={'data': data})
shutil.rmtree(folder)
def mpi_run(custom_command=""):
"""
Launch a simple MPI run, with no communication of covariance matrix
It simply allows the first process to create the folder - so that the
log.param is properly written. A signal is then send to the other
processes, that contains the chain number of the parent run.
In order to be sure to have different chain numbers, it adds the rank of
the process and the initial job number - this should avoid conflict, but
can be subject of future improvements
"""
from mpi4py import MPI
comm = MPI.COMM_WORLD
nprocs = comm.Get_size()
rank = comm.Get_rank()
# this will be the master cpu. This guy will create - or append - a folder,
# being sure to be the first to do so.
if rank == 0:
# First initialisation
cosmo, data, command_line, success = safe_initialisation(
custom_command, comm, nprocs)
# Check that the run asked is compatible with mpirun and prepare.
if command_line.subparser_name == 'info':
warnings.warn(
"Analyzing the chains is not supported in mpirun"
" so this will run on one core only.")
status = 'failed'
elif command_line.method == "MH":
regexp = re.match(".*__(\w*).txt", data.out_name)
suffix = regexp.groups()[0]
status = suffix
elif command_line.method == "NS":
status = 1
else:
warnings.warn(
"The method '%s' is not supported"%(command_line.method) +
" in mpirun so this will run on one core only.")
status = 'failed'
# Send an "OK" signal to all the other processes, actually giving the
# suffix of this master chain. All the other will add 1 to this number
for index in range(1, nprocs):
comm.send(status, dest=index, tag=1)
else:
# If the rank is not 0, it is a slave process. It waits to receive the
# "OK" message, which is immediatly discarded.
status = comm.recv(source=0, tag=1)
# If a failed message was passed, exit the process
if status == 'failed':
success = False
else:
# Concatenate the rank to the suffix, and not the opposite, this
# should avoid any conflicting name
if not custom_command:
custom_command = " ".join(sys.argv[1:])
suffix = status
custom_command += " --chain-number %s" % str(int(suffix)+rank)
cosmo, data, command_line, success = initialise(custom_command)
if success:
import sampler
sampler.run(cosmo, data, command_line)
success = comm.gather(success, root=0)
return
# calling the parent method initialization
dict.__init__(self)
self["initial"] = array[0:4]
self["scale"] = array[4]
self["role"] = array[-1]
self["tex_name"] = io_mp.get_tex_name(key)
if array[3] == 0:
self["status"] = "fixed"
self["current"] = array[0]
else:
self["status"] = "varying"
self["prior"] = prior.Prior(array)
class Container(object):
"""Dummy class to act as a namespace for data"""
pass
if __name__ == "__main__":
import doctest
import shutil
from initialise import initialise
folder = os.path.join("tests", "doc")
cosmo, data, command_line, _ = initialise("-o %s -p test.param" % folder)
doctest.testmod(extraglobs={"data": data})
shutil.rmtree(folder)
import numpy as np
import constant
import initialise
import interpolate
import preprocess
import save
if __name__ == '__main__':
algorithm_start_time = time()
print('Initialize the algorithm...')
start_time = time()
action, training_files, testing_files, squares, features = initialise.initialise(
)
squares_training_data = preprocess.load_squares(training_files, squares)
squares_testing_data = preprocess.load_squares(testing_files, squares)
print('Done to initialze the algorithm. Time: ',
round(time() - start_time, 3), ' sec')
for square in squares:
print('*** SQUARE ' + str(square) + ' ***')
if (square
not in squares_training_data) or (square
not in squares_testing_data):
print('No data for square (' + str(square) + '), skip.')
def init():
"""Initialise tracking tools and database."""
initialise.initialise()
# path for all necessary input
paths = {}
# cosmological code path (CLASS)
paths['cosmo'] = '/net/eemmeer/data1/ssli/class_public'
# father path for all the input and output
paths['data'] = '/disks/shear15/ssli/CosmicShear'
# parameter/configure file path
paths[
'param'] = '/net/raam/data1/surfdrive_ssli/Projects/6CosmicShear_RB/CosmicShearRB/Cosmo/cosmic_shear_signal/input'
# +++++++++++++++++++ Running scripts
Start = time.time()
# Initialisation
# class: data and cosmo created
cosmo, data = initialise.initialise(paths)
for i in range(len(name_param_files)):
# parameter file (with cosmological and nuisance parameters)
name_param_file = name_param_files[i]
data.read_file(name_param_file, 'data', field='', separate=False)
# output folder
data.conf['out_folder'] = out_folder
data.conf['out_suffix'] = out_suffix_s[i]
# # ++++++++++++++++++++++++++++++++++++++++++ whole
# # name of configure files
# name_conf_file = 'kv450_cf.conf'
# # data filled with input files
# # Number of bins
# nzbins = len(data_whole.const['z_bins_min'])
# # Number of correlation
# nzcorrs = int(nzbins * (nzbins + 1) / 2)
# #
# chi2_whole, dof_whole = Chi2Test.Chi2SingleFunc(nzbins, nzcorrs, theta_bins_whole, mask_whole, data_whole, xi_obs_whole, xi_theo_whole)
# # print("chi2_whole", chi2_whole)
# # print("dof_whole", dof_whole)
# ++++++++++++++++++++++++++++++++++++++++++ red
# name of parameter/configure files
name_conf_file = 'kv450_cf_red.conf'
# Initialisation
# class: data and cosmo created
cosmo, data_red = initialise.initialise(paths)
# data filled with input files
# parameter file (with cosmological and nuisance parameters)
data_red.read_file(name_param_file, 'data', field='', separate=False)
# configure file (with configure and hardly changed setting parameters)
data_red.read_file(name_conf_file, 'data', field='', separate=False)
# cosmic shear signal calculation
xi_obs_red, xi_theo_red, theta_bins_red, mask_red = CosmicShear.CSsignalFunc(
data_red, cosmo, save_theory_vector=False)
# xi_obs_red, xi_theo_red, theta_bins_red, mask_red = CosmicShear.CSsignalFunc(data_red, cosmo, True)
# chi2
# Number of bins
nzbins = len(data_red.const['z_bins_min'])
# Number of correlation