def main(): for i in range(0,10): tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) for j in range(0,10): nameMock = str(i) + "_" + str(j) tmp_command = "echo \" \n ------ Mock: " + str(i) + " " + str(j) + " ------\n \" " subprocess.call(tmp_command, shell=True) tmp_command = "clubatch \"echo ; hostname ; " + commandProd + str(i) + " " + str(j) + "\"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) myTools.isReadyForNewJobs(150, 430) time.sleep(5)
def main():
### For mocks
### mock catalogue
for i in range(0,1):
### mock catalogue
for j in range(0,1):
### for region
#for k in range(0,80):
### for metals
#for k in [21,24,25,26,27]:
for k in range(21,29):
tmp_command = "echo \" \n ------ Start ------ \n \" "
subprocess.call(tmp_command, shell=True)
tmp_command = "clubatch \"time ; hostname ; " + commandProd + ' 405 0 ' + str(k) + ' 0 ' + str(i) + " " + str(j) + "\""
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + tmp_command
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + time.ctime()
subprocess.call(tmp_command, shell=True)
time.sleep(0.5)
myTools.isReadyForNewJobs(200, 1000,'time')
time.sleep(120)
'''
for i in range(0,29):
for j in range(0,29):
tmp_command = "echo \" \n ------ Start ------ \n \" "
subprocess.call(tmp_command, shell=True)
tmp_command = "clubatch \"time ; hostname ; " + commandProd + " 20 " + str(i) + ' ' + str(j) + " 0 0 0 \""
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + tmp_command
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + time.ctime()
subprocess.call(tmp_command, shell=True)
time.sleep(1)
myTools.isReadyForNewJobs(200, 430,'time')
'''
'''
### for metals templates
for k in range(0,29):
tmp_command = "echo \" \n ------ Start ------ \n \" "
subprocess.call(tmp_command, shell=True)
tmp_command = "clubatch \"time ; hostname ; " + commandProd + ' 15 0 ' + str(k) + " 0 0 0 \""
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + tmp_command
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + time.ctime()
subprocess.call(tmp_command, shell=True)
time.sleep(10)
myTools.isReadyForNewJobs(100, 430,'time')
'''
'''
for i in range(0,10):
for j in range(0,10):
### Get the data to 'good' files
#####################
command = "clubatch \" time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/Python/Correlation/main.py " + str(i) + ' ' + str(j) + "\""
print command
subprocess.call(command, shell=True)
time.sleep(0.1)
time.sleep(0.1)
'''
'''
def main():
stepIdx = 1
for i in range(0,1):
tmp_command = "echo \" \n ------ Start ------ \n \" "
subprocess.call(tmp_command, shell=True)
for j in range(0,1):
nameMock = str(i) + "_" + str(j)
tmp_command = "echo \" \n ------ Mock: " + str(i) + " " + str(j) + " ------\n \" "
subprocess.call(tmp_command, shell=True)
if (stepIdx==0):
tmp_command = "clubatch \"echo ; hostname ; " + commandProd + str(i) + " " + str(j) + " 0 0 \""
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + tmp_command
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + time.ctime()
subprocess.call(tmp_command, shell=True)
myTools.isReadyForNewJobs(120, 430)
time.sleep(200)
if (stepIdx==1):
### Do the fits
step = 2000
first = 0
last = step
while (first <= nbSpectra):
tmp_command = "clubatch \"echo ; hostname ; " + commandProd + str(i) + " " + str(j) + " " + str(first) + " " + str(last) +"\""
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + tmp_command
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + time.ctime()
subprocess.call(tmp_command, shell=True)
first = first + step
last = last + step
myTools.isReadyForNewJobs(150, 430)
time.sleep(1.)
if (stepIdx==2):
### Put files of alpha and beta together
folder = '/home/gpfs/manip/mnt0607/bao/hdumasde/Results/Txt/chain_annalys_delta/'
scheme = 'alphaAndBeta_'
lenScheme = len(scheme)
endScheme = '_'+str(i)+'_'+str(j)+'.txt'
lenEndScheme = -len(endScheme)
tmp_command = "rm " + folder + scheme + 'all.txt'
subprocess.call(tmp_command, shell=True)
### Get the list of files
all_t = os.listdir(folder)
tmp_all_t = []
for el in all_t:
if (el[:lenScheme]==scheme and el[lenEndScheme:]==endScheme):
tmp_all_t.append(el)
all_t = tmp_all_t
### Sort the list of files
convert = lambda text: int(text) if text.isdigit() else text
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
all_t = sorted(all_t, key = alphanum_key)
for el in all_t:
print el
data = numpy.loadtxt(folder+all_t[0])
idx = data[:,0].astype(int)
alpha = data[:,1]
beta = data[:,2]
chi = data[:,3]
for el in all_t[1:]:
fo = open(folder+el, "r")
print folder+el
if os.fstat(fo.fileno()).st_size:
data = numpy.loadtxt(folder+el)
idx = numpy.append( idx, data[:,0].astype(int) )
alpha = numpy.append( alpha, data[:,1] )
beta = numpy.append( beta, data[:,2] )
chi = numpy.append( chi, data[:,3] )
print idx
print alpha
print beta
print idx.size
numpy.savetxt(folder+scheme+'all.txt', zip(idx,alpha,beta,chi))
'''
plt.hist(alpha,bins=100,label='alpha')
plt.show()
plt.hist(beta,bins=100,label='beta')
plt.show()
plt.hist(chi[ (numpy.isfinite(chi)) ],bins=100,label='beta')
plt.show()
'''
### Put the new alpha and beta in fits file
path = '/home/gpfs/manip/mnt0607/bao/hdumasde/MockV4/M3_0_'+str(i)+'/00'+str(j)+'/mock.fits'
print path
file_cat = pyfits.open(path,mode='update')
cat = file_cat[1].data
print cat['ALPHA_2']
cat['ALPHA_2'] = alpha
cat['BETA_2'] = beta
print cat['ALPHA_2']
file_cat.close()
print
print " ------ End ------"
print
def sendCalculDelta():
stepIdx = 3
tmp_command = "echo \" \n ------ Start ------ \n \" "
subprocess.call(tmp_command, shell=True)
for i in range(0,10):
path = pathToFolder + 'M3_0_' + str(i) + '/'
for j in range(0,10):
tmp_command = "echo " + str(i) + " " + str(j)
subprocess.call(tmp_command, shell=True)
path = pathToFolder+ 'M3_0_' + str(i) + '/00'+str(j) + '/'
if (stepIdx==0):
### Get the data to 'good' files
#####################
command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + ' ' + str(j) + "\""
print command
subprocess.call(command, shell=True)
myTools.isReadyForNewJobs(10, 430)
time.sleep(10)
if (stepIdx==1):
### Do the fits
nbSpectra = 200000
step = 2000
first = 0
last = step
while (first <= nbSpectra):
tmp_command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + " " + str(j) + " " + str(first) + " " + str(last) +"\""
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + tmp_command
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + time.ctime()
subprocess.call(tmp_command, shell=True)
first = first + step
last = last + step
myTools.isReadyForNewJobs(150, 430)
time.sleep(1.)
if (stepIdx==2):
### Put files of alpha and beta together
folder = '/home/gpfs/manip/mnt0607/bao/hdumasde/Results/Txt/chain_annalys_delta/'
scheme = 'alphaAndBeta_'
lenScheme = len(scheme)
endScheme = '_'+str(i)+'_'+str(j)+'.txt'
lenEndScheme = -len(endScheme)
tmp_command = "rm " + folder + scheme + 'all.txt'
subprocess.call(tmp_command, shell=True)
### Get the list of files
all_t = os.listdir(folder)
tmp_all_t = []
for el in all_t:
if (el[:lenScheme]==scheme and el[lenEndScheme:]==endScheme):
tmp_all_t.append(el)
all_t = tmp_all_t
### Sort the list of files
convert = lambda text: int(text) if text.isdigit() else text
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
all_t = sorted(all_t, key = alphanum_key)
for el in all_t:
print el
data = numpy.loadtxt(folder+all_t[0])
idx = data[:,0].astype(int)
alpha = data[:,1]
beta = data[:,2]
chi = data[:,3]
for el in all_t[1:]:
fo = open(folder+el, "r")
print folder+el
if os.fstat(fo.fileno()).st_size:
data = numpy.loadtxt(folder+el)
if (data.size/4 != 1):
idx = numpy.append( idx, data[:,0].astype(int) )
alpha = numpy.append( alpha, data[:,1] )
beta = numpy.append( beta, data[:,2] )
chi = numpy.append( chi, data[:,3] )
else:
idx = numpy.append( idx, data[0].astype(int) )
alpha = numpy.append( alpha, data[1] )
beta = numpy.append( beta, data[2] )
chi = numpy.append( chi, data[3] )
print idx
print alpha
print beta
print idx.size
numpy.savetxt(folder+scheme+'all.txt', zip(idx,alpha,beta,chi))
'''
plt.hist(alpha,bins=100,label='alpha')
plt.show()
plt.hist(beta,bins=100,label='beta')
plt.show()
plt.hist(chi[ (numpy.isfinite(chi)) ],bins=100,label='beta')
plt.show()
'''
### Put the new alpha and beta in fits file
path = path + 'Data/delta.fits'
print path
file_cat = pyfits.open(path,mode='update')
cat = file_cat[1].data
print cat['ALPHA_2']
cat['ALPHA_2'] = alpha
cat['BETA_2'] = beta
print cat['ALPHA_2']
file_cat.close()
if (stepIdx==3):
### Get the data to 'good' files
#####################
command = "clubatch \"time ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/chain_annalys_delta/Correlation/bin/main.exe 5 0 0 0 " + str(i) + ' ' + str(j) + "\""
print command
subprocess.call(command, shell=True)
time.sleep(10)
if (stepIdx==4):
### Get the data to 'good' files
#####################
command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Python/Correlation/xi_delta_QSO.py a a a a 0 " + str(i) + ' ' + str(j) + "\""
print command
subprocess.call(command, shell=True)
if (stepIdx==5):
### Get the data to 'good' files
#####################
command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Python/Correlation/xi_Q_Q.py a a a a " + str(i) + ' ' + str(j) + "\""
print command
subprocess.call(command, shell=True)
time.sleep(120)
def main():
print ' index_pass = '******'mkdir ' +pathToFolder+'Results/', shell=True)
subprocess.call('mkdir ' +pathToFolder+'Results_no_metals/', shell=True)
subprocess.call('mkdir ' +pathToFolder+'Results_no_projection/', shell=True)
subprocess.call('mkdir ' +pathToFolder+'Results_raw_from_JeanMarc/', shell=True)
subprocess.call('mkdir ' +pathToFolder+'Results_delta_gaussian/', shell=True)
subprocess.call('mkdir ' +pathToFolder+'Results_only_LR/', shell=True)
subprocess.call('mkdir ' +pathToFolder+'Results_only_LR_noRand/', shell=True)
subprocess.call('mkdir ' +pathToFolder+'Results_only_LR_noRand_noRSD/', shell=True)
for i in range(0,10):
path = pathToFolder + 'Box_00' + str(i) + '/'
## Create the folders
#####################
if (index_pass==0):
subprocess.call('mkdir ' +path, shell=True)
for j in range(0,10):
#if (i==0 and j==0): continue
print i, j
path = pathToFolder+ 'Box_00' + str(i) + '/Simu_00'+str(j) + '/'
if (index_pass==0):
subprocess.call('mkdir ' + path, shell=True)
subprocess.call('mkdir ' + path + 'Raw', shell=True)
subprocess.call('mkdir ' + path + 'Data', shell=True)
subprocess.call('mkdir ' + path + 'Data_no_metals', shell=True)
subprocess.call('mkdir ' + path + 'Run', shell=True)
subprocess.call('mkdir ' + path + 'Run_no_metals', shell=True)
subprocess.call('mkdir ' + path + 'Results/', shell=True)
subprocess.call('mkdir ' + path + 'Results_no_metals/', shell=True)
subprocess.call('mkdir ' + path + 'Results_no_projection/', shell=True)
subprocess.call('mkdir ' + path + 'Results_raw_from_JeanMarc/', shell=True)
subprocess.call('mkdir ' + path + 'Results_delta_gaussian/', shell=True)
subprocess.call('mkdir ' + path + 'Results_only_LR/', shell=True)
subprocess.call('mkdir ' + path + 'Results_only_LR_noRand/', shell=True)
subprocess.call('mkdir ' + path + 'Results_only_LR_noRand_noRSD/', shell=True)
elif (index_pass==1):
'''
/home/gpfs/manip/mnt0607/bao/hdumasde/Program/LyAMockExpander/Expand.sh -i /home/gpfs/manip/mnt0607/bao/jmlg/QSOlyaMocks/v1575/fits/spectra-7850-0.fits -o /home/gpfs/manip/mnt0607/bao/hdumasde/Mock_JMLG/v1575_with_good_metals/Box_000/Simu_000/Raw/ -columns loglam,flux,ivar,and_mask,mock_contpca,mock_F,mock_Fmet -JM -vac /home/gpfs/manip/mnt0607/bao/hdumasde/Data/Catalogue/DR12Q_v2_10.fits -data /home/gpfs/manip/mnt0607/bao/Spectra/SpectraV5_8_guy/spectra -seed 0 -metals 2
'''
command = '/home/gpfs/manip/mnt0607/bao/hdumasde/Program/LyAMockExpander/Expand.sh -i /home/gpfs/manip/mnt0607/bao/jmlg/QSOlyaMocks/v1575/fits/spectra-785'+str(i)+'-'+str(j)+'.fits -o ' +path+ 'Raw/ -columns loglam,flux,ivar,and_mask,mock_contpca,mock_F,mock_Fmet -JM -vac /home/gpfs/manip/mnt0607/bao/hdumasde/Data/Catalogue/DR12Q_v2_10.fits -data /home/gpfs/manip/mnt0607/bao/Spectra/SpectraV5_8_guy/spectra -metals 2 -seed ' + str(j)+ ' -seed_first_index ' + str(i)
command = "clubatch \"echo ; hostname ; "+ command + "\""
print command
subprocess.call("echo " + command, shell=True)
subprocess.call(command, shell=True)
myTools.isReadyForNewJobs(20, 1000,'echo')
time.sleep(60)
elif (index_pass==2):
if (i==0 and j==0):
tbhduQSO = create_fits_qso(sizeMax)
tbhduQSO.writeto(path + 'Data/QSO_noRSD.fits', clobber=True)
#tbhduForest = create_fits_forest(sizeMaxForest)
#tbhduForest.writeto(path + 'Data/delta.fits', clobber=True)
#tbhduForest.writeto(path + 'Data_no_metals/delta.fits', clobber=True)
else:
#command = 'cp ' + pathToFolder + 'Box_000/Simu_000/Data/delta.fits ' + path + 'Data/delta.fits'
#subprocess.call(command, shell=True)
#command = 'cp ' + pathToFolder + 'Box_000/Simu_000/Data/delta.fits ' + path + 'Data_no_metals/delta.fits'
#subprocess.call(command, shell=True)
command = 'cp ' + pathToFolder + 'Box_000/Simu_000/Data/QSO_noRSD.fits ' + path + 'Data/QSO_noRSD.fits'
subprocess.call(command, shell=True)
#myTools.isReadyForNewJobs(10, 430,'cp')
#time.sleep(30)
elif (index_pass==3):
command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/Mock_JMLG/ReadFits/bin/main.exe " + str(i) + ' ' + str(j)
command = "clubatch \"echo ; hostname ; "+ command + "\""
print command
subprocess.call(command, shell=True)
myTools.isReadyForNewJobs(10, 1000,'echo')
time.sleep(10)
elif (index_pass==4):
'''
print path + 'Data/QSO_withRSD.fits'
cat = pyfits.open(path + 'Data/QSO_withRSD.fits')[1].data
nbQSO = cat.size
print ' nb of QSOs = ', nbQSO
'''
cat = pyfits.open(path + 'Data/QSO_noRSD.fits', memmap=True)[1].data
print ' nb of QSOs before = ', cat.size
cat = cat[ (cat['Z'] != 0.) ]
print ' nb of QSOs after = ', cat.size
nbQSO = cat.size
pyfits.writeto(path + 'Data/QSO_noRSD.fits', cat, clobber=True)
"""
### Remove useless lines in Forest
cat = pyfits.open(path + 'Data_no_metals/delta.fits', memmap=True)[1].data
print ' nb of forests before = ', cat.size
cat = cat[ (cat['Z'] != 0.) ]
nbFor = cat.size
print ' nb of forests after = ', nbFor
if ( nbFor>nbFor__ ):
print nbFor-int(nbQSO*ratioForestToQSO__)
rand = numpy.random.choice(nbFor, nbFor-int(nbQSO*ratioForestToQSO__), replace=False)
cat['Z'][rand] = 0.
print ' nb of forests before = ', cat.size
cat = cat[ (cat['Z'] != 0.) ]
print ' nb of forests after = ', cat.size
pyfits.writeto(path + 'Data_no_metals/delta.fits', cat, clobber=True)
"""
'''
### Get the data to 'good' files
#####################
#command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Mock_JMLG/launchjob.sh " + str(i) + ' ' + str(j) + "\""
command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Mock_JMLG/ReadFits/bin/main.exe " + str(i) + ' ' + str(j) + "\""
print command
subprocess.call(command, shell=True)
myTools.isReadyForNewJobs(10, 430)
time.sleep(30)
'''
print
print " ------ End ------"
print
def sendCalculDelta():
tmp_command = "echo \" \n ------ Start ------ \n \" "
subprocess.call(tmp_command, shell=True)
for i in range(0,10):
path = pathToFolder + 'Box_00' + str(i) + '/'
for j in range(0,10):
tmp_command = "echo " + str(i) + " " + str(j)
subprocess.call(tmp_command, shell=True)
path = pathToFolder + 'Box_00' + str(i) + '/Simu_00'+str(j) + '/'
if (index_pass==0):
### Get the data to 'good' files
#####################
command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + ' ' + str(j) + " 0 0 2 0"
command = "clubatch \"echo ; hostname ; "+command + "\""
print command
subprocess.call(command, shell=True)
myTools.isReadyForNewJobs(10, 1000,'echo')
time.sleep(30)
if (index_pass==1):
### Do the fits
nbSpectra = 200000
step = 2000
first = 0
last = step
while (first <= nbSpectra):
tmp_command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + " " + str(j) + " " + str(first) + " " + str(last) +" 2 1 \""
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + tmp_command
subprocess.call(tmp_command, shell=True)
tmp_command = "echo " + time.ctime()
subprocess.call(tmp_command, shell=True)
first = first + step
last = last + step
myTools.isReadyForNewJobs(100, 1000,'echo')
time.sleep(0.2)
if (index_pass==2):
command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + ' ' + str(j) + " 0 0 2 2"
command = "clubatch \"echo ; hostname ; "+command + "\""
print command
subprocess.call(command, shell=True)
myTools.isReadyForNewJobs(20, 1000,'echo')
time.sleep(30)
if (index_pass==3):
command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Correlation/bin/main.exe 2 0 0 0 " + str(i) + ' ' + str(j)
command = "clubatch \"echo ; hostname ; "+command + "\""
print command
subprocess.call(command, shell=True)
myTools.isReadyForNewJobs(30, 1000,'echo')
time.sleep(30)
if (index_pass==4):
### Get the data to 'good' files
#####################
command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Python/Correlation/xi_delta_QSO.py a a a a 0 " + str(i) + ' ' + str(j) + "\""
print command
time.sleep(1.)
subprocess.call(command, shell=True)
if (index_pass==5):
### Get the data to 'good' files
#####################
command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/Python/Correlation/main_Q.py " + str(i) + ' ' + str(j) + "\""
print command
subprocess.call(command, shell=True)
time.sleep(10.)
