def __init__(self, mle_folders, mle_parameters, cluster_parameters, \
cluster_folders, sim_parameters):
self.datafile_path_dict = \
{'asymptotic': mle_folders.get_path('current_output_subfolder'),
'sim_based': mle_folders.get_path('sim_profile_fixed_pt_folder')}
self.LL_profile_folder = mle_folders.get_path('LL_list_path')
self.mle_parameters = copy.deepcopy(mle_parameters)
self.mle_folders = copy.deepcopy(mle_folders)
self.parameter_holder_dict = \
{'asymptotic': copy.deepcopy(mle_parameters),
'sim_based': copy.deepcopy(sim_parameters)}
self.cluster_parameters = copy.deepcopy(cluster_parameters)
self.cluster_folders = copy.deepcopy(cluster_folders)
self.runtime_percentile = mle_parameters.get_option(
'runtime_percentile')
self.pval = mle_parameters.get_option('CI_pval')
self._create_combined_output_file()
self.max_LL = None
self.warnings = CombinedResultWarning()
self.CI_type_list = ['asymptotic', 'sim_based']
self.completeness_tracker = cluster_functions.CompletenessTracker(['asymptotic_initialization', \
'asymptotic_CIs', 'sim_based_initialization', 'sim_based_CIs'])
self.runtime_quant_list = np.array([])
self.true_max_param_df = pd.DataFrame()
self.combined_results_df = pd.DataFrame()
# set completefiles for asymptotic and sim-based CIs
self._create_completefile_dict()
# check current completeness of CIs
self._check_CI_completeness()
# set MLE results from 'unfixed' parameter (i.e. fitting all
# unfixed params together)
self.unfixed_mle_file = \
generate_filename(self.datafile_path_dict['asymptotic'], \
'1', mle_parameters.get_option('output_identifier'), 'unfixed', 'data')
def __init__(self, pval, LL_df, deg_freedom, fixed_param_MLE_val, \
fixed_param, CI_type, mle_folders, cluster_parameters, \
cluster_folders, output_identifier):
self.CI_sides = ['lower','upper']
self.CI_object_dictionary = dict()
self.CI_dictionary = dict()
self.CI_completeness_tracker = cluster_functions.CompletenessTracker(self.CI_sides)
self.CI_complete = False
self.CI_warning_list = []
self.CI_object_dictionary = {}
self.CI_object_dictionary['lower'] = OneSidedCIBoundLower(pval/2, \
LL_df, deg_freedom, fixed_param_MLE_val, fixed_param, \
CI_type, mle_folders, cluster_parameters, cluster_folders, \
output_identifier)
self.CI_object_dictionary['upper'] = OneSidedCIBoundUpper(pval/2, \
LL_df, deg_freedom, fixed_param_MLE_val, fixed_param, \
CI_type, mle_folders, cluster_parameters, cluster_folders, \
output_identifier)
def __init__(self, parameter_input):
self.model_list = parameter_input["model"]
self.model_completeness_tracker = \
cluster_functions.CompletenessTracker(self.model_list)
self.all_models_complete = False
required_mle_key_list = ['input_datafile_keys', \
'input_datafile_values', 'model', 'parameter_list', \
'top_level_parameters', 'permafixed_parameters', \
'starting_parameter_vals', 'min_parameter_vals', \
'max_parameter_vals', 'multistart_positions', \
'multistart_grid_parameters', 'logspace_profile_parameters', \
'scaling_array', 'x_tolerance', 'fun_tolerance', \
'LL_calculator', 'pre_MLE_function', 'post_MLE_function', \
'gradient_specification']
required_CI_key_list = ['profile_point_num_list', \
'profile_lower_limits', 'profile_upper_limits', 'CI_pval', \
'runtime_percentile']
complete_required_key_list = \
required_mle_key_list + required_CI_key_list
super(MLEParameters, self).__init__(parameter_input, \
complete_required_key_list)
def set_model(self, model_name, output_identifier):
# for all MLE_parameter attributes, retrieves the parameter or
# list of parameters corresponding to the current model
self.current_model_complete = False
# output_identifier is a string that will be included in filenames
current_option_dict = {'output_identifier': output_identifier, \
'current_model': model_name}
model_idx = self.model_list.index(model_name)
number_of_models = len(self.input_val_dict['model'])
self.current_option_dict = self._select_sublist(self.input_val_dict, \
current_option_dict, model_idx, number_of_models)
# check that each parameter for current model is in parameter_list once
if len(self.current_option_dict['parameter_list']) > \
len(set(self.current_option_dict['parameter_list'])):
raise AttributeError('Parameter list for model ' + model_name + \
'contains non-unique parameters: ' + \
str(self.current_option_dict['parameter_list']))
# identify list of parameters that are permanently fixed
permafixed_parameter_bool = \
[x in self.current_option_dict['permafixed_parameters'] \
for x in self.current_option_dict['parameter_list']]
self.current_option_dict['permafixed_parameter_bool'] = \
permafixed_parameter_bool
# for permafixed parameters, set lower and upper profile pt limits to starting_val
# profile_point_num needs to be reset too, but that can happen when specifying
self.current_option_dict['profile_lower_limits']\
[permafixed_parameter_bool] = \
self.current_option_dict['starting_parameter_vals']\
[permafixed_parameter_bool]
self.current_option_dict['profile_upper_limits']\
[permafixed_parameter_bool] = \
self.current_option_dict['starting_parameter_vals']\
[permafixed_parameter_bool]
# identify parameters MLE needs to be performed on
self._id_parameters_to_loop_over()
# set up completefile tracker for these parameters
self.parameter_completeness_tracker = \
cluster_functions.CompletenessTracker(\
self.current_option_dict['parameters_to_loop_over'])
def generate_CIs(self, CI_type):
''' Generates and record CIs for fitted parameters '''
if CI_type not in self.CI_type_list:
raise ValueError('Unknown CI type ' + CI_type)
parameter_holder = self.parameter_holder_dict[CI_type]
# initialize combined results df
self._initialize_asymptotic_results()
# # check whether sim_based profiles ready
# self._initialize_sim_based_results()
# Only run CI estimation if initialization is complete but CIs are not
if self.completeness_tracker.get_key_completeness(CI_type + \
'_initialization'):
if not \
self.completeness_tracker.get_key_completeness(CI_type + \
'_CIs'):
if CI_type == 'asymptotic':
parameters_to_loop_over = \
parameter_holder.get_fitted_parameter_list(False)
elif CI_type == 'sim_based':
parameters_to_loop_over = \
parameter_holder.get_option('sim_CI_parameters')
CI_completeness_tracker = \
cluster_functions.CompletenessTracker(parameters_to_loop_over)
if parameters_to_loop_over:
self._read_combined_df()
for current_fixed_parameter in parameters_to_loop_over:
# check whether this CI has been identified
# read line for current_fixed_parameter from
# combined_results_df
current_results_line = SingleParamResultSummary()
current_results_line.read_from_df_line(self.combined_results_df, \
current_fixed_parameter)
# identify columns containing the string
# (CI_type + '_CI') and check that such columns
# exist and that none of them contain NaN)
current_CI_complete = \
current_results_line.check_column_filledness_by_keyword(CI_type + '_CI')
if current_CI_complete:
CI_completeness_tracker.switch_key_completeness(current_fixed_parameter, \
True)
else:
# first set current parameter
parameter_holder.set_parameter(
current_fixed_parameter)
# create an LLProfile for current parameter
ll_profile = mle_functions.LLProfile(parameter_holder, \
self.datafile_path_dict[CI_type], \
self.LL_profile_folder, \
pd.DataFrame())
ll_profile.run_LL_list_compilation()
ll_profile.run_CI(self.pval, self.mle_folders, \
self.cluster_parameters, self.cluster_folders, \
parameter_holder, CI_type)
CI_dict = ll_profile.get_CI()
# if jobs to calculate both CI bounds have not yet been
# completed, returns None
if CI_dict:
# set the new confidence interval
current_results_line.set_CI(CI_type, CI_dict)
# replace previous warning entry--all warnings are
# recalculated individually when a CI bound is
# initialized, so having a CI_bound completely
# identified previously will not cause a problem
# in skipping warnings
current_warning = ll_profile.get_warnings()
current_results_line.set_warnings(
current_warning)
# replace line in combined_results_df with updated line
# that has CI and warnings
current_results_dict = current_results_line.get_contents(
)
self._add_line_to_combined_df(
current_fixed_parameter,
current_results_dict)
CI_completeness_tracker.switch_key_completeness(
current_fixed_parameter, True)
self._write_combined_df()
CIs_just_completed = \
CI_completeness_tracker.get_completeness()
if CIs_just_completed:
open(self.CI_completefile_dict[CI_type], 'a').close()
############### ??? TO DO ??? ###############
# Update file types being taken in
# Check for data analysis complete_file before running anything in this loop
############### ??? TO DO ??? ###############
# current_input_data_prefix = os.path.join(cluster_folders.experiment_path, \
# (current_growth_condition + '_'))
##########
# Run MLE - fixed and unfixed
# Run 2 reps - one 'fast' with standard grid_power and L,
# and a second one to ensure same results when
# grid_power and L are increased
rep_completeness_tracker = cluster_functions.CompletenessTracker(
rep_list)
for rep_index, rep in enumerate(rep_list):
rep_float = float(rep)
# set L and gridpower values for FFT
# current_L = parameter_list["starting_L"]*pow(1.5,rep_float-1)
# returns starting_L for rep1 and starting_L*1.5 for rep2
# current_gridpower = parameter_list["starting_gridpower"]+(rep_float-1)
# set memory and time for current run
cluster_parameters.set_current_time(rep_float * \
cluster_parameters.get_input_option('starting_time'))
cluster_parameters.set_current_mem(rep_float * \
cluster_parameters.get_input_option('starting_mem'))
# set subfolder to be in current rep