def main(): for i in range(0,10): tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) for j in range(0,10): nameMock = str(i) + "_" + str(j) tmp_command = "echo \" \n ------ Mock: " + str(i) + " " + str(j) + " ------\n \" " subprocess.call(tmp_command, shell=True) tmp_command = "clubatch \"echo ; hostname ; " + commandProd + str(i) + " " + str(j) + "\"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) myTools.isReadyForNewJobs(150, 430) time.sleep(5)
def main(): ### For mocks ### mock catalogue for i in range(0,1): ### mock catalogue for j in range(0,1): ### for region #for k in range(0,80): ### for metals #for k in [21,24,25,26,27]: for k in range(21,29): tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) tmp_command = "clubatch \"time ; hostname ; " + commandProd + ' 405 0 ' + str(k) + ' 0 ' + str(i) + " " + str(j) + "\"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) time.sleep(0.5) myTools.isReadyForNewJobs(200, 1000,'time') time.sleep(120) ''' for i in range(0,29): for j in range(0,29): tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) tmp_command = "clubatch \"time ; hostname ; " + commandProd + " 20 " + str(i) + ' ' + str(j) + " 0 0 0 \"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) time.sleep(1) myTools.isReadyForNewJobs(200, 430,'time') ''' ''' ### for metals templates for k in range(0,29): tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) tmp_command = "clubatch \"time ; hostname ; " + commandProd + ' 15 0 ' + str(k) + " 0 0 0 \"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) time.sleep(10) myTools.isReadyForNewJobs(100, 430,'time') ''' ''' for i in range(0,10): for j in range(0,10): ### Get the data to 'good' files ##################### command = "clubatch \" time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/Python/Correlation/main.py " + str(i) + ' ' + str(j) + "\"" print command subprocess.call(command, shell=True) time.sleep(0.1) time.sleep(0.1) ''' '''
def main(): stepIdx = 1 for i in range(0,1): tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) for j in range(0,1): nameMock = str(i) + "_" + str(j) tmp_command = "echo \" \n ------ Mock: " + str(i) + " " + str(j) + " ------\n \" " subprocess.call(tmp_command, shell=True) if (stepIdx==0): tmp_command = "clubatch \"echo ; hostname ; " + commandProd + str(i) + " " + str(j) + " 0 0 \"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) myTools.isReadyForNewJobs(120, 430) time.sleep(200) if (stepIdx==1): ### Do the fits step = 2000 first = 0 last = step while (first <= nbSpectra): tmp_command = "clubatch \"echo ; hostname ; " + commandProd + str(i) + " " + str(j) + " " + str(first) + " " + str(last) +"\"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) first = first + step last = last + step myTools.isReadyForNewJobs(150, 430) time.sleep(1.) if (stepIdx==2): ### Put files of alpha and beta together folder = '/home/gpfs/manip/mnt0607/bao/hdumasde/Results/Txt/chain_annalys_delta/' scheme = 'alphaAndBeta_' lenScheme = len(scheme) endScheme = '_'+str(i)+'_'+str(j)+'.txt' lenEndScheme = -len(endScheme) tmp_command = "rm " + folder + scheme + 'all.txt' subprocess.call(tmp_command, shell=True) ### Get the list of files all_t = os.listdir(folder) tmp_all_t = [] for el in all_t: if (el[:lenScheme]==scheme and el[lenEndScheme:]==endScheme): tmp_all_t.append(el) all_t = tmp_all_t ### Sort the list of files convert = lambda text: int(text) if text.isdigit() else text alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] all_t = sorted(all_t, key = alphanum_key) for el in all_t: print el data = numpy.loadtxt(folder+all_t[0]) idx = data[:,0].astype(int) alpha = data[:,1] beta = data[:,2] chi = data[:,3] for el in all_t[1:]: fo = open(folder+el, "r") print folder+el if os.fstat(fo.fileno()).st_size: data = numpy.loadtxt(folder+el) idx = numpy.append( idx, data[:,0].astype(int) ) alpha = numpy.append( alpha, data[:,1] ) beta = numpy.append( beta, data[:,2] ) chi = numpy.append( chi, data[:,3] ) print idx print alpha print beta print idx.size numpy.savetxt(folder+scheme+'all.txt', zip(idx,alpha,beta,chi)) ''' plt.hist(alpha,bins=100,label='alpha') plt.show() plt.hist(beta,bins=100,label='beta') plt.show() plt.hist(chi[ (numpy.isfinite(chi)) ],bins=100,label='beta') plt.show() ''' ### Put the new alpha and beta in fits file path = '/home/gpfs/manip/mnt0607/bao/hdumasde/MockV4/M3_0_'+str(i)+'/00'+str(j)+'/mock.fits' print path file_cat = pyfits.open(path,mode='update') cat = file_cat[1].data print cat['ALPHA_2'] cat['ALPHA_2'] = alpha cat['BETA_2'] = beta print cat['ALPHA_2'] file_cat.close() print print " ------ End ------" print
def sendCalculDelta(): stepIdx = 3 tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) for i in range(0,10): path = pathToFolder + 'M3_0_' + str(i) + '/' for j in range(0,10): tmp_command = "echo " + str(i) + " " + str(j) subprocess.call(tmp_command, shell=True) path = pathToFolder+ 'M3_0_' + str(i) + '/00'+str(j) + '/' if (stepIdx==0): ### Get the data to 'good' files ##################### command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + ' ' + str(j) + "\"" print command subprocess.call(command, shell=True) myTools.isReadyForNewJobs(10, 430) time.sleep(10) if (stepIdx==1): ### Do the fits nbSpectra = 200000 step = 2000 first = 0 last = step while (first <= nbSpectra): tmp_command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + " " + str(j) + " " + str(first) + " " + str(last) +"\"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) first = first + step last = last + step myTools.isReadyForNewJobs(150, 430) time.sleep(1.) if (stepIdx==2): ### Put files of alpha and beta together folder = '/home/gpfs/manip/mnt0607/bao/hdumasde/Results/Txt/chain_annalys_delta/' scheme = 'alphaAndBeta_' lenScheme = len(scheme) endScheme = '_'+str(i)+'_'+str(j)+'.txt' lenEndScheme = -len(endScheme) tmp_command = "rm " + folder + scheme + 'all.txt' subprocess.call(tmp_command, shell=True) ### Get the list of files all_t = os.listdir(folder) tmp_all_t = [] for el in all_t: if (el[:lenScheme]==scheme and el[lenEndScheme:]==endScheme): tmp_all_t.append(el) all_t = tmp_all_t ### Sort the list of files convert = lambda text: int(text) if text.isdigit() else text alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] all_t = sorted(all_t, key = alphanum_key) for el in all_t: print el data = numpy.loadtxt(folder+all_t[0]) idx = data[:,0].astype(int) alpha = data[:,1] beta = data[:,2] chi = data[:,3] for el in all_t[1:]: fo = open(folder+el, "r") print folder+el if os.fstat(fo.fileno()).st_size: data = numpy.loadtxt(folder+el) if (data.size/4 != 1): idx = numpy.append( idx, data[:,0].astype(int) ) alpha = numpy.append( alpha, data[:,1] ) beta = numpy.append( beta, data[:,2] ) chi = numpy.append( chi, data[:,3] ) else: idx = numpy.append( idx, data[0].astype(int) ) alpha = numpy.append( alpha, data[1] ) beta = numpy.append( beta, data[2] ) chi = numpy.append( chi, data[3] ) print idx print alpha print beta print idx.size numpy.savetxt(folder+scheme+'all.txt', zip(idx,alpha,beta,chi)) ''' plt.hist(alpha,bins=100,label='alpha') plt.show() plt.hist(beta,bins=100,label='beta') plt.show() plt.hist(chi[ (numpy.isfinite(chi)) ],bins=100,label='beta') plt.show() ''' ### Put the new alpha and beta in fits file path = path + 'Data/delta.fits' print path file_cat = pyfits.open(path,mode='update') cat = file_cat[1].data print cat['ALPHA_2'] cat['ALPHA_2'] = alpha cat['BETA_2'] = beta print cat['ALPHA_2'] file_cat.close() if (stepIdx==3): ### Get the data to 'good' files ##################### command = "clubatch \"time ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/chain_annalys_delta/Correlation/bin/main.exe 5 0 0 0 " + str(i) + ' ' + str(j) + "\"" print command subprocess.call(command, shell=True) time.sleep(10) if (stepIdx==4): ### Get the data to 'good' files ##################### command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Python/Correlation/xi_delta_QSO.py a a a a 0 " + str(i) + ' ' + str(j) + "\"" print command subprocess.call(command, shell=True) if (stepIdx==5): ### Get the data to 'good' files ##################### command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Python/Correlation/xi_Q_Q.py a a a a " + str(i) + ' ' + str(j) + "\"" print command subprocess.call(command, shell=True) time.sleep(120)
def main(): print ' index_pass = '******'mkdir ' +pathToFolder+'Results/', shell=True) subprocess.call('mkdir ' +pathToFolder+'Results_no_metals/', shell=True) subprocess.call('mkdir ' +pathToFolder+'Results_no_projection/', shell=True) subprocess.call('mkdir ' +pathToFolder+'Results_raw_from_JeanMarc/', shell=True) subprocess.call('mkdir ' +pathToFolder+'Results_delta_gaussian/', shell=True) subprocess.call('mkdir ' +pathToFolder+'Results_only_LR/', shell=True) subprocess.call('mkdir ' +pathToFolder+'Results_only_LR_noRand/', shell=True) subprocess.call('mkdir ' +pathToFolder+'Results_only_LR_noRand_noRSD/', shell=True) for i in range(0,10): path = pathToFolder + 'Box_00' + str(i) + '/' ## Create the folders ##################### if (index_pass==0): subprocess.call('mkdir ' +path, shell=True) for j in range(0,10): #if (i==0 and j==0): continue print i, j path = pathToFolder+ 'Box_00' + str(i) + '/Simu_00'+str(j) + '/' if (index_pass==0): subprocess.call('mkdir ' + path, shell=True) subprocess.call('mkdir ' + path + 'Raw', shell=True) subprocess.call('mkdir ' + path + 'Data', shell=True) subprocess.call('mkdir ' + path + 'Data_no_metals', shell=True) subprocess.call('mkdir ' + path + 'Run', shell=True) subprocess.call('mkdir ' + path + 'Run_no_metals', shell=True) subprocess.call('mkdir ' + path + 'Results/', shell=True) subprocess.call('mkdir ' + path + 'Results_no_metals/', shell=True) subprocess.call('mkdir ' + path + 'Results_no_projection/', shell=True) subprocess.call('mkdir ' + path + 'Results_raw_from_JeanMarc/', shell=True) subprocess.call('mkdir ' + path + 'Results_delta_gaussian/', shell=True) subprocess.call('mkdir ' + path + 'Results_only_LR/', shell=True) subprocess.call('mkdir ' + path + 'Results_only_LR_noRand/', shell=True) subprocess.call('mkdir ' + path + 'Results_only_LR_noRand_noRSD/', shell=True) elif (index_pass==1): ''' /home/gpfs/manip/mnt0607/bao/hdumasde/Program/LyAMockExpander/Expand.sh -i /home/gpfs/manip/mnt0607/bao/jmlg/QSOlyaMocks/v1575/fits/spectra-7850-0.fits -o /home/gpfs/manip/mnt0607/bao/hdumasde/Mock_JMLG/v1575_with_good_metals/Box_000/Simu_000/Raw/ -columns loglam,flux,ivar,and_mask,mock_contpca,mock_F,mock_Fmet -JM -vac /home/gpfs/manip/mnt0607/bao/hdumasde/Data/Catalogue/DR12Q_v2_10.fits -data /home/gpfs/manip/mnt0607/bao/Spectra/SpectraV5_8_guy/spectra -seed 0 -metals 2 ''' command = '/home/gpfs/manip/mnt0607/bao/hdumasde/Program/LyAMockExpander/Expand.sh -i /home/gpfs/manip/mnt0607/bao/jmlg/QSOlyaMocks/v1575/fits/spectra-785'+str(i)+'-'+str(j)+'.fits -o ' +path+ 'Raw/ -columns loglam,flux,ivar,and_mask,mock_contpca,mock_F,mock_Fmet -JM -vac /home/gpfs/manip/mnt0607/bao/hdumasde/Data/Catalogue/DR12Q_v2_10.fits -data /home/gpfs/manip/mnt0607/bao/Spectra/SpectraV5_8_guy/spectra -metals 2 -seed ' + str(j)+ ' -seed_first_index ' + str(i) command = "clubatch \"echo ; hostname ; "+ command + "\"" print command subprocess.call("echo " + command, shell=True) subprocess.call(command, shell=True) myTools.isReadyForNewJobs(20, 1000,'echo') time.sleep(60) elif (index_pass==2): if (i==0 and j==0): tbhduQSO = create_fits_qso(sizeMax) tbhduQSO.writeto(path + 'Data/QSO_noRSD.fits', clobber=True) #tbhduForest = create_fits_forest(sizeMaxForest) #tbhduForest.writeto(path + 'Data/delta.fits', clobber=True) #tbhduForest.writeto(path + 'Data_no_metals/delta.fits', clobber=True) else: #command = 'cp ' + pathToFolder + 'Box_000/Simu_000/Data/delta.fits ' + path + 'Data/delta.fits' #subprocess.call(command, shell=True) #command = 'cp ' + pathToFolder + 'Box_000/Simu_000/Data/delta.fits ' + path + 'Data_no_metals/delta.fits' #subprocess.call(command, shell=True) command = 'cp ' + pathToFolder + 'Box_000/Simu_000/Data/QSO_noRSD.fits ' + path + 'Data/QSO_noRSD.fits' subprocess.call(command, shell=True) #myTools.isReadyForNewJobs(10, 430,'cp') #time.sleep(30) elif (index_pass==3): command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/Mock_JMLG/ReadFits/bin/main.exe " + str(i) + ' ' + str(j) command = "clubatch \"echo ; hostname ; "+ command + "\"" print command subprocess.call(command, shell=True) myTools.isReadyForNewJobs(10, 1000,'echo') time.sleep(10) elif (index_pass==4): ''' print path + 'Data/QSO_withRSD.fits' cat = pyfits.open(path + 'Data/QSO_withRSD.fits')[1].data nbQSO = cat.size print ' nb of QSOs = ', nbQSO ''' cat = pyfits.open(path + 'Data/QSO_noRSD.fits', memmap=True)[1].data print ' nb of QSOs before = ', cat.size cat = cat[ (cat['Z'] != 0.) ] print ' nb of QSOs after = ', cat.size nbQSO = cat.size pyfits.writeto(path + 'Data/QSO_noRSD.fits', cat, clobber=True) """ ### Remove useless lines in Forest cat = pyfits.open(path + 'Data_no_metals/delta.fits', memmap=True)[1].data print ' nb of forests before = ', cat.size cat = cat[ (cat['Z'] != 0.) ] nbFor = cat.size print ' nb of forests after = ', nbFor if ( nbFor>nbFor__ ): print nbFor-int(nbQSO*ratioForestToQSO__) rand = numpy.random.choice(nbFor, nbFor-int(nbQSO*ratioForestToQSO__), replace=False) cat['Z'][rand] = 0. print ' nb of forests before = ', cat.size cat = cat[ (cat['Z'] != 0.) ] print ' nb of forests after = ', cat.size pyfits.writeto(path + 'Data_no_metals/delta.fits', cat, clobber=True) """ ''' ### Get the data to 'good' files ##################### #command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Mock_JMLG/launchjob.sh " + str(i) + ' ' + str(j) + "\"" command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Mock_JMLG/ReadFits/bin/main.exe " + str(i) + ' ' + str(j) + "\"" print command subprocess.call(command, shell=True) myTools.isReadyForNewJobs(10, 430) time.sleep(30) ''' print print " ------ End ------" print
def sendCalculDelta(): tmp_command = "echo \" \n ------ Start ------ \n \" " subprocess.call(tmp_command, shell=True) for i in range(0,10): path = pathToFolder + 'Box_00' + str(i) + '/' for j in range(0,10): tmp_command = "echo " + str(i) + " " + str(j) subprocess.call(tmp_command, shell=True) path = pathToFolder + 'Box_00' + str(i) + '/Simu_00'+str(j) + '/' if (index_pass==0): ### Get the data to 'good' files ##################### command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + ' ' + str(j) + " 0 0 2 0" command = "clubatch \"echo ; hostname ; "+command + "\"" print command subprocess.call(command, shell=True) myTools.isReadyForNewJobs(10, 1000,'echo') time.sleep(30) if (index_pass==1): ### Do the fits nbSpectra = 200000 step = 2000 first = 0 last = step while (first <= nbSpectra): tmp_command = "clubatch \"echo ; hostname ; /home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + " " + str(j) + " " + str(first) + " " + str(last) +" 2 1 \"" subprocess.call(tmp_command, shell=True) tmp_command = "echo " + tmp_command subprocess.call(tmp_command, shell=True) tmp_command = "echo " + time.ctime() subprocess.call(tmp_command, shell=True) first = first + step last = last + step myTools.isReadyForNewJobs(100, 1000,'echo') time.sleep(0.2) if (index_pass==2): command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Get_delta/bin/main.exe " + str(i) + ' ' + str(j) + " 0 0 2 2" command = "clubatch \"echo ; hostname ; "+command + "\"" print command subprocess.call(command, shell=True) myTools.isReadyForNewJobs(20, 1000,'echo') time.sleep(30) if (index_pass==3): command = "/home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/chain_annalys_delta/Correlation/bin/main.exe 2 0 0 0 " + str(i) + ' ' + str(j) command = "clubatch \"echo ; hostname ; "+command + "\"" print command subprocess.call(command, shell=True) myTools.isReadyForNewJobs(30, 1000,'echo') time.sleep(30) if (index_pass==4): ### Get the data to 'good' files ##################### command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/Python/Correlation/xi_delta_QSO.py a a a a 0 " + str(i) + ' ' + str(j) + "\"" print command time.sleep(1.) subprocess.call(command, shell=True) if (index_pass==5): ### Get the data to 'good' files ##################### command = "clubatch \"time ; hostname ; python /home/gpfs/manip/mnt0607/bao/hdumasde/Code/CrossCorrelation/Python/Correlation/main_Q.py " + str(i) + ' ' + str(j) + "\"" print command subprocess.call(command, shell=True) time.sleep(10.)