def __init__(self,params,out_path_orig,pymc_import=False,multinest_import=False):
params.out_save_plots = True
if params.gen_type == 'transmission' or params.fit_transmission:
if os.path.isdir(os.path.join(out_path_orig, 'stage_0')):
params.out_path = os.path.join(out_path_orig, 'stage_0')
else:
params.out_path = out_path_orig
dataob = data(params)
atmosphereob = atmosphere(dataob)
forwardmodelob = transmission(atmosphereob)
fittingob = fitting(forwardmodelob)
if params.mcmc_run and pymc_import:
fittingob.MCMC = True
if params.nest_run and multinest_import:
fittingob.NEST = True
outputob = output(fittingob)
if params.verbose or params.out_save_plots:
outputob.plot_all(save2pdf=params.out_save_plots, param_labels=fittingob.fit_params_texlabels)
outputob.save_ascii_spectra()
if params.gen_type == 'emission' or params.fit_emission:
folders = ['stage_0', 'stage_1']
for f in folders:
dir = os.path.join(out_path_orig, f)
if os.path.isdir(dir):
params.out_path = dir
dataob = data(params)
if f is 'stage_1':
Cov_array = np.loadtxt(os.path.join(out_path_orig, 'stage_0/tp_covariance.dat'))
atmosphereob = atmosphere(dataob, tp_profile_type='hybrid', covariance=Cov_array)
else:
atmosphereob = atmosphere(dataob)
forwardmodelob = emission(atmosphereob)
fittingob = fitting(forwardmodelob)
if params.mcmc_run and pymc_import:
fittingob.MCMC = True
if params.nest_run and multinest_import:
fittingob.NEST = True
outputob = output(fittingob)
if params.verbose or params.out_save_plots:
outputob.plot_all(save2pdf=params.out_save_plots,
params_names=fittingob.fit_params_names[:fittingob.fit_X_nparams],
params_labels=fittingob.fit_params_texlabels[:fittingob.fit_X_nparams])
outputob.save_ascii_spectra()
# save and plot TP profile (plotting only if save2pdf=True)
outputob.save_TP_profile(save2pdf=True) #saving TP profile
#deleting objectes
dataob = None; atmosphereob = None; forwardmodelob = None; fittingob = None;
outputob = None
del dataob; del atmosphereob; del forwardmodelob; del fittingob; del outputob;
def run(params, options=False):
# initialising data object
dataob = data(params)
#initialising TP profile instance
atmosphereob = atmosphere(dataob)
#initialising transmission radiative transfer code instance
forwardmodelob = transmission(atmosphereob)
#initialising fitting object
fittingob = fitting(forwardmodelob)
#fit data
if params.downhill_run:
if MPIimport:
if MPI.COMM_WORLD.Get_rank() == 0:
fittingob.downhill_fit(
) # simplex downhill fit, only on first core
else:
fittingob.downhill_fit(
) # simplex downhill fit, only on first core
if MPIimport:
MPI.COMM_WORLD.Barrier() # wait for everybody to synchronize here
if params.mcmc_run and pymc_import:
fittingob.mcmc_fit() # MCMC fit
if MPIimport:
MPI.COMM_WORLD.Barrier()
if (not options and params.nest_run and multinest_import) \
or (params.nest_run and multinest_import and not options.no_multinest):
fittingob.multinest_fit() # Nested sampling fit
elif options and (params.nest_run and multinest_import
and options.no_multinest):
fittingob.NEST = True
if params.nest_poly_run and polychord_import:
fittingob.polychord_fit() #Polychord sampling fit
if MPIimport:
MPI.COMM_WORLD.Barrier() # wait for everybody to synchronize here
# exit if the rank of MPI process is > 0 (.e. leave only master process running)
if MPIimport:
MPIsize = MPI.COMM_WORLD.Get_size()
if MPI.COMM_WORLD.Get_rank() > 0:
sys.exit()
else:
MPIsize = 0
# initiating output instance with fitted data from fitting class
# running inteprolations with nthreads = MPIsize
outputob = output(fittingob)
return outputob
def init_stats(self):
'''
manually initialises the TauREx output analysis
when not loaded distance functinos are still available
'''
self.dir = self.options.dir
self.params.gen_manual_waverange = False
self.params.nest_run = False
self.params.mcmc_run = False
self.params.downhill_run = True
#setting up output storage
self.stats = {}
#loading data from TauREx NEST output
self.load_traces_likelihood('nest_out.db')
#loading parameter list
# self.parameters = np.loadtxt(os.path.join(self.dir,'parameters.txt'),dtype='str')
self.stats['parameters'] = self.NEST_db['fit_params_names']
# initialising data object
self.dataob = data(self.params)
# initialising atmosphere object
self.atmosphereob = atmosphere(self.dataob)
# set forward model
if self.params.gen_type == 'transmission':
self.fmob = transmission(self.atmosphereob)
elif self.params.gen_type == 'emission':
self.fmob = emission(self.atmosphereob)
#initialising fitting object
self.fitting = fitting(self.fmob)
def run(params, options=False):
out_path_orig = params.out_path
###############################
# STAGE 1
###############################
# set output directory of stage 1
params.out_path = os.path.join(out_path_orig, 'stage_0')
# initialising data object
dataob = data(params)
#initialising TP profile instance
atmosphereob = atmosphere(dataob)
#initialising emission radiative transfer code instance
forwardmodelob = emission(atmosphereob, stage=0)
#initialising fitting object
fittingob = fitting(forwardmodelob)
#fit data for stage 1
if params.downhill_run:
fittingob.downhill_fit() #simplex downhill fit
if params.mcmc_run and pymc_import:
fittingob.mcmc_fit() # MCMC fit
if MPIimport:
MPI.COMM_WORLD.Barrier() # wait for everybody to synchronize here
if params.nest_run and multinest_import:
fittingob.multinest_fit() # Nested sampling fit
if MPIimport:
MPI.COMM_WORLD.Barrier() # wait for everybody to synchronize here
if MPIimport and MPI.COMM_WORLD.Get_rank() != 0:
exit()
outputob = output(fittingob,
out_path=os.path.join(out_path_orig, 'stage_0'))
return outputob
# exit()
# todo fix stage 2
# generating TP profile covariance from previous fit
Cov_array = generate_tp_covariance(outputob)
# saving covariance
if MPIimport and MPI.COMM_WORLD.Get_rank() is 0 or MPIimport is False:
np.savetxt(os.path.join(params.out_path, 'tp_covariance.dat'),
Cov_array)
###############################
# STAGE 2
###############################
if params.fit_emission_stage2:
# set output directory of stage 2
params.out_path = os.path.join(out_path_orig, 'stage_1')
#setting up objects for stage 2 fitting
dataob1 = data(params)
#setting stage 2 atmosphere object
atmosphereob1 = atmosphere(dataob1,
tp_profile_type='hybrid',
covariance=Cov_array)
#setting stage 2 forward model
forwardmodelob1 = emission(atmosphereob1)
#setting stage 2 fitting object
fittingob1 = fitting(forwardmodelob1)
# #running stage 2 fit
if params.downhill_run:
fittingob1.downhill_fit() #simplex downhill fit
if params.mcmc_run and pymc_import:
fittingob1.mcmc_fit() # MCMC fit
MPI.COMM_WORLD.Barrier() # wait for everybody to synchronize here
if params.nest_run and multinest_import:
fittingob1.multinest_fit() # Nested sampling fit
MPI.COMM_WORLD.Barrier() # wait for everybody to synchronize here
###############
#finished fitting. Post fitting analysis from here
#forcing slave processes to exit at this stage
if MPIimport and MPI.COMM_WORLD.Get_rank() != 0:
exit()
#initiating output instance with fitted data from fitting class
if params.fit_emission_stage2:
outputob1 = output(fittingob1,
out_path=os.path.join(out_path_orig, 'stage_1'))
def runModel():
log.info('POST /run')
try:
if request.form and request.form['jsonData']:
parameters = json.loads(request.form['jsonData'])
else:
message = "Missing input jsonData!"
log.error(message)
return buildFailure(message, 400)
sendToQueue = parameters.get('sendToQueue', False)
inputFileName = None
id = str(uuid.uuid4())
if (len(request.files) > 0):
inputCSVFile = request.files['csvFile']
ext = os.path.splitext(inputCSVFile.filename)[1]
if sendToQueue:
bucket = S3Bucket(INPUT_BUCKET, log)
object = bucket.uploadFileObj(getInputFileKey(id, ext),
inputCSVFile)
if object:
parameters['inputCSVFile'] = {
'originalName': inputCSVFile.filename,
'bucket_name': object.bucket_name,
'key': object.key
}
else:
message = "Upload CSV file to S3 failed!"
log.error(message)
return buildFailure(message, 500)
else:
parameters['inputCSVFile'] = inputCSVFile.filename
inputFileName = getInputFilePath(id, ext)
inputCSVFile.save(inputFileName)
if not os.path.isfile(inputFileName):
message = "Upload file failed!"
log.error(message)
return buildFailure(message, 500)
outputRdsFileName = getOutputFilePath(id, '.rds')
outputSSFileName = getOutputFilePath(
id, extensionMap[SS_FILE_TYPE])
outputFileName = getOutputFilePath(id, '.out')
parameters['filename'] = inputFileName
parameters['outputRdsFilename'] = outputRdsFileName
parameters['outputFilename'] = outputFileName
else:
message = 'No input data (CSV) file, please upload a data file!'
log.warning(message)
return buildFailure(message, 400)
columns = [
parameters['outcomeC'], parameters['outcomeL'],
parameters['outcomeR']
]
if 'design' in parameters and parameters['design'] == 1:
columns += [parameters['strata'], parameters['weight']]
parameters['weightInfo'] = [{
'samp.weight': parameters['weight'],
'strata': parameters['strata']
}]
if parameters['covariatesSelection']:
covariateNameMap = mapCategoricalCovariates(
parameters['covariatesArr'])
columns += parameters['covariatesSelection']
covariates = ' + '.join([
covariateNameMap[x] for x in parameters['covariatesSelection']
])
if 'effects' in parameters:
effects = [
covariateNameMap[x[0]] + ' * ' + covariateNameMap[x[1]]
for x in parameters['effects']
]
effectsPlain = [
x[0] + ' * ' + x[1] for x in parameters['effects']
]
if effects:
covariates += ' + ' + ' + '.join(effects)
parameters['effectsString'] = ' + '.join(effectsPlain)
parameters['covariates'] = covariates
parameters['columns'] = columns
if sendToQueue:
# Send parameters to queue
sqs = Queue(log)
sqs.sendMsgToQueue(
{
'parameters': parameters,
'jobId': id,
'extension': ext,
'jobType': 'fitting'
}, id)
return buildSuccess({
'enqueued':
True,
'jobId':
id,
'message':
'Job "{}" has been added to queue successfully!'.format(
parameters.get('jobName', 'PIMixture'))
})
else:
fittingResult = fitting(parameters,
outputSSFileName,
SS_FILE_TYPE,
log,
timeout=FITTING_TIMEOUT)
if fittingResult['status']:
return buildSuccess(fittingResult['results'])
else:
return buildFailure(fittingResult)
except Exception as e:
exc_type, exc_obj, tb = sys.exc_info()
f = tb.tb_frame
lineno = tb.tb_lineno
inputFileName = f.f_code.co_filename
linecache.checkcache(inputFileName)
line = linecache.getline(inputFileName, lineno, f.f_globals)
log.exception("Exception occurred")
return buildFailure({
"status": False,
"message": "An unknown error occurred"
})