def pipeline(cat, config, data, binstatus, binno=None, mccount=0):
disp, spectrum, stackspectrum, analysis, analysisfinal = None, None, None, None, None
fig, ax1, ax2, ax3, ax4 = startfigure(cat)
if cat.progress == 'progress':
plt.show(block=False)
else:
pass
## creating the stack objects
xs = cat.spectralaxis.value[0]
xe = cat.spectralaxis.value[-1]
if uf.checkkeys(data, 'StackEntireCatalogueYN'):
stacknoYN = data['StackEntireCatalogueYN']
else:
stacknoYN = raw_input(
'Do you want to stack your entire catalogue? [y/n] ')
if stacknoYN.lower() == 'y':
stackobj = len(cat.catalogue)
else:
if uf.checkkeys(data, 'NumberCatalogueObjects'):
stackobj = int(data['NumberCatalogueObjects'])
else:
stackobj = None
while type(stackobj) != int:
stackobj = int(
input(
'Please enter the number of profiles you want to stack: '
))
if stackobj > cat.numcatspec:
stackobj = len(cat.catalogue)
else:
h = 0
logging.info('Number of objects going into the stack: %i' % stackobj)
if cat.uncert == 'y' and mccount == 0:
if binstatus == 'bin':
print('\nProgress of first round of Bin %i: ' % binno)
logging.info('Starting first round of Bin %i.' % binno)
else:
print('\nProgress of first round: ')
logging.info('Starting stacking first round.')
elif cat.uncert == 'n':
if binstatus == 'bin':
print('\nStacking progress of Bin %i: ' % binno)
logging.info('Starting stacking of Bin %i.' % binno)
else:
print('\nStacking progress: ')
logging.info('Starting stacking.')
else:
h = 0
pbar = ''
if ((mccount == 0) or ((mccount == 1)) and (binstatus == 'bin')):
cat, analysisfinal = stackeverything(pbar, cat, stackobj, binstatus,
'firstcall', binno, mccount, fig,
ax1, ax2, ax3, ax4)
else:
h = 0
if mccount == 0 and cat.uncert == 'y' and binstatus == 'bin':
logging.info('Finished stacking first round of Bin %i.' % binno)
return cat
else:
disp = DD.dispData(cat)
mccount += 1
if cat.uncert == 'y':
if binstatus == 'bin':
print('\n\nUncertainty calculations of Bin %i in progress:' %
binno)
else:
print('\n\nUncertainty calculations in progress:')
pbar = ''
if cat.multiop == 'multi':
results = []
cpu_count = int(multi.cpu_count() / 2)
logging.info(
'Spawning %i processes to run uncertainty calculations.' %
cpu_count)
pool = multi.Pool(processes=cpu_count, maxtasksperchild=1)
else:
h = 0
analysisempty = UA.uncertAnalysis(cat,
analysisfinal,
allspec=[],
allnoise=[],
allrms=[],
middle=[],
allintflux=np.array([[], [], [],
[], [], [],
[], []]))
mccount = 1
suppress = cat.suppress
cat.suppress = 'hide'
fullcatfile = cat.catalogue
logging.info('Starting uncertainty calculations...')
while mccount < (cat.mc + 1):
if cat.uncerttype == 'dagjk':
indices = np.random.randint(low=0,
high=(len(fullcatfile) - 1),
size=int(cat.R *
len(fullcatfile)))
cat.catalogue = fullcatfile[indices]
if stackobj > len(cat.catalogue):
stackobj = len(cat.catalogue)
else:
h = 0
else:
h = 0
if mccount > 0 and cat.multiop != 'multi':
frac = int((mccount) / (cat.mc * 2.) * 100.)
if frac < 2:
frac = 0
else:
h = 0
pbar = uf.progressbar(frac, pbar)
cat = stackeverything(pbar,
cat,
stackobj,
binstatus,
'uncertcall',
binno,
mccount,
fig,
ax1,
ax2,
ax3,
ax4,
analysisfinal=analysisfinal)
else:
result = pool.apply_async(
stackeverything,
args=(pbar, cat, stackobj, binstatus, 'uncertcall',
binno, mccount, fig, ax1, ax2, ax3, ax4,
analysisfinal))
results.append(result)
mccount += 1
if cat.multiop == 'multi':
pool.close()
pbar = ''
for n in range(cat.mc):
frac = int((n + 1.) / cat.mc * 100.)
if frac < 2:
frac = 0
else:
h = 0
pbar = uf.progressbar(frac, pbar)
results[n].get()
analysisfinal = analysisfinal + analysisempty.load(
n + 1, cat)
else:
for n in range(1, cat.mc + 1, 1):
frac = int(float(cat.mc + n) / (cat.mc * 2.) * 100.)
pbar = uf.progressbar(frac, pbar)
analysisfinal = analysisfinal + analysisempty.load(n, cat)
cat.suppress = suppress
analysisfinal.ProcessUncertainties(cat)
pck.dump(
analysisfinal,
open('%sstackedresultsobj_%s.pkl' % (cat.outloc, cat.runtime),
'wb'))
disp.display(cat, analysisfinal.allintflux,
[analysisfinal.middle, analysisfinal.allrms],
args.suppress)
else:
disp.display(cat, None, cat.suppress)
return cat
def stackeverything(pbar,
cat,
stackobj,
binstatus,
callno,
binno,
mccount,
fig,
ax1,
ax2,
ax3,
ax4,
analysisfinal=None):
stackspectrum = SD.objStack(spec=np.zeros(cat.exspeclen) * cat.stackunit,
ref=np.zeros(cat.exspeclen) * cat.stackunit,
noise=np.zeros(cat.noiselen),
weight=0.,
spectral=cat.spectralaxis,
sunits=cat.stackunit)
colours = uf.colourgen(stackobj)
## loop through all of spectra listed in the catalogue
######################################################################################
## Stack the catalogue
######################################################################################
for n in range(0, stackobj):
spectrum = SD.objSpec()
col = colours[n]
cat = spectrum.callModule(cat, n, mccount, axes=[ax1, ax2], fig=fig)
if spectrum.origspec is None:
continue
else:
stackspectrum = stackspectrum + spectrum
fig = plotprogress(cat, stackobj, col, stackspectrum, spectrum,
fig, ax1, ax2, ax3, ax4, n)
if mccount == 0:
frac = int((n + 1.) / stackobj * 100.)
if frac < 2:
frac = 0
else:
h = 0
pbar = uf.progressbar(frac, pbar)
if cat.saveprogress == 'save':
outloc = cat.outloc + 'progressplots/'
uf.checkpath(outloc)
if binstatus == 'bin':
plt.savefig(
outloc + 'bin%i_window_%i_source_%s.png' %
(binno, n, str(cat.catalogue['Object ID'][n])),
bbox_inches='tight',
pad_inches=0.2)
else:
plt.savefig(outloc + 'window_%i_source_%s.png' %
(n, str(cat.catalogue['Object ID'][n])),
bbox_inches='tight',
pad_inches=0.2)
else:
h = 0
else:
h = 0
######################################################################################
######################################################################################
# close the progress window before moving on to analysis
if cat.progress == 'progress' and mccount == 0:
time.sleep(5)
plt.close('all')
elif cat.saveprogress == 'save':
plt.close('all')
else:
cat.progress = None
######################################################################################
## Abort process if the stack is empty
######################################################################################
if len(stackspectrum.stackrms) == 0 and len(
cat.outcatalogue) == 0 and cat.uncert == 'n':
print(
"\nThere were no spectra to stack or something went wrong. I am going to abort and you need to check that there are spectrum files in the correct folder and that the spectrum file names are correctly entered in to the the catalogue."
)
logging.critical(
'There were no spectra to stack, or something else went wrong. Please check that the spectrum files in specified folder.'
)
uf.earlyexit(cat)
elif len(stackspectrum.stackrms) == 0 and len(
cat.outcatalogue) != 0 and cat.uncert == 'n':
print(
"\nThere were no spectra to stack or something went wrong. I am going to abort and you need to check that there are spectrum files in the correct folder and that the spectrum file names are correctly entered in to the the catalogue."
)
return cat
elif len(stackspectrum.stackrms) == 0 and len(
cat.outcatalogue) == 0 and cat.uncert == 'y':
return cat, None
######################################################################################
## Continue on to analysis
######################################################################################
else:
analysis = AD.anaData()
cat = analysis.analyse_data(stackspectrum, cat, mccount)
##################################################################################
## If Uncertainty needs to run..
##################################################################################
stackspectrum, spectrum = None, None
if cat.uncert == 'n' or callno == 'firstcall':
analysisfinal = UA.uncertAnalysis(cat,
analysis,
allspec=[],
allnoise=[],
allrms=[],
middle=[],
allintflux=np.array([[], [], [],
[], [], [],
[], []]))
return cat, analysisfinal
else:
analysisfinal.updateStackAnalysis(cat, mccount, analysis)
analysis = None
return cat
return cat