def automatic_fit_light_curve(light_curve_df, minimum_score=0.3, plots_save_path=None, verbose=False, logger=None): """Automatically fit the best support vector machine regression (SVR) model for the input light curve. Inputs: light_curve_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex, and columns for irradiance and uncertainty. Optional Inputs: minimum_score [float]: Set this to the minimum explained variance score (0 - 1) acceptable for fits. If the best fit score is < minimum_score, this function will return np.nan for light_curve_fit. Default value is 0.3. plots_save_path [str]: Set to a path in order to save the validation curve and best fit overplot on the data to disk. Default is None, meaning no plots will be saved to disk. verbose [bool]: Set to log the processing messages to disk and console. Default is False. logger [JpmLogger]: A configured logger from jpm_logger.py. If set to None, will generate a new one. Default is None. Outputs: light_curve_fit_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex, and columns for fitted irradiance and uncertainty. best_fit_gamma [float]: The best found gamma hyper parameter for the SVR. best_fit_score [float]: The best explained variance score. Optional Outputs: None Example: light_curve_fit, best_fit_gamma, best_fit_score = automatic_fit_light_curve(light_curve_df, verbose=True) """ # Prepare the logger for verbose if verbose: if not logger: logger = JpmLogger(filename='automatic_fit_light_curve_log', path='/Users/jmason86/Desktop/') logger.info("Running on event with light curve start time of {0}.".format(light_curve_df.index[0])) # Pull data out of the DataFrame for compatibility formatting X = metatimes_to_seconds_since_start(light_curve_df.index) y = light_curve_df['irradiance'].values # Check for NaNs and issue warning that they are being removed from the dataset if verbose: if np.isnan(y).any(): logger.warning("There are NaN values in light curve. Dropping them.") finite_irradiance_indices = np.isfinite(y) X = X[finite_irradiance_indices] X = X.reshape(len(X), 1) # Format to be compatible with validation_curve and SVR.fit() uncertainty = light_curve_df.uncertainty[np.isfinite(y)] y = y[finite_irradiance_indices] if verbose: logger.info("Fitting %s points." % len(y)) # Helper function for compatibility with validation_curve def jpm_svr(gamma=1e-6, **kwargs): return make_pipeline(SVR(kernel='rbf', C=1e3, gamma=gamma, **kwargs)) # Hyper parameter for SVR is gamma, so generate values of it to try gamma = np.logspace(-7, 1, num=20, base=10) # Overwrite the default scorer (R^2) with explained variance score evs = make_scorer(explained_variance_score) # Split the data between training/testing 50/50 but across the whole time range rather than the default consecutive Kfolds import time t0 = time.time() shuffle_split = ShuffleSplit(n_splits=20, train_size=0.5, test_size=0.5, random_state=None) # Generate the validation curve -- test all them gammas! # Parallelized to speed it up (n_jobs = # of parallel threads) train_score, val_score = validation_curve(jpm_svr(), X, y, 'svr__gamma', gamma, cv=shuffle_split, n_jobs=7, scoring=evs) t1 = time.time() logger.error('It took {0} seconds to run.'.format(t1 - t0)) if verbose: logger.info("Validation curve complete.") if plots_save_path: plt.clf() plt.style.use('jpm-transparent-light') plt.plot(gamma, np.median(train_score, 1), label='training score') plt.plot(gamma, np.median(val_score, 1), label='validation score') ax = plt.axes() plt.legend(loc='best') plt.title("t$_0$ = " + datetimeindex_to_human(light_curve_df.index)[0]) ax.set_xscale('log') plt.xlabel('gamma') plt.ylabel('score') plt.ylim(0, 1) filename = plots_save_path + 'Validation Curve t0 ' + datetimeindex_to_human(light_curve_df.index)[0] + '.png' plt.savefig(filename) if verbose: logger.info("Validation curve saved to %s" % filename) # Identify the best score scores = np.median(val_score, axis=1) best_fit_score = np.max(scores) best_fit_gamma = gamma[np.argmax(scores)] if verbose: logger.info('Scores: ' + str(scores)) logger.info('Best score: ' + str(best_fit_score)) logger.info('Best fit gamma: ' + str(best_fit_gamma)) # Return np.nan if only got bad fits if best_fit_score < minimum_score: if verbose: logger.warning("Uh oh. Best fit score {0:.2f} is < user-defined minimum score {1:.2f}".format(best_fit_score, minimum_score)) return np.nan, best_fit_gamma, best_fit_score # Otherwise train and fit the best model sample_weight = 1 / uncertainty model = SVR(kernel='rbf', C=1e3, gamma=best_fit_gamma).fit(X, y, sample_weight) y_fit = model.predict(X) if verbose: logger.info("Best model trained and fitted.") if plots_save_path: plt.clf() plt.errorbar(X.ravel(), y, yerr=uncertainty, color='black', fmt='o', label='Input light curve') plt.plot(X.ravel(), y_fit, linewidth=6, label='Fit') plt.title("t$_0$ = " + datetimeindex_to_human(light_curve_df.index)[0]) plt.xlabel('time [seconds since start]') plt.ylabel('irradiance [%]') plt.legend(loc='best') filename = plots_save_path + 'Fit t0 ' + datetimeindex_to_human(light_curve_df.index)[0] + '.png' plt.savefig(filename) if verbose: logger.info("Fitted curve saved to %s" % filename) # TODO: Get uncertainty of fit at each point... if that's even possible # Placeholder for now just so that the function can complete: output uncertainty = input uncertainty fit_uncertainty = uncertainty # Construct a pandas DataFrame with DatetimeIndex, y_fit, and fit_uncertainty light_curve_fit_df = pd.DataFrame({'irradiance': y_fit, 'uncertainty': fit_uncertainty}) light_curve_fit_df.index = light_curve_df.index[finite_irradiance_indices] if verbose: logger.info("Created output DataFrame") return light_curve_fit_df, best_fit_gamma, best_fit_score
def generate_jedi_catalog( flare_index_range, # =range(0, 5052), threshold_time_prior_flare_minutes=480.0, dimming_window_relative_to_flare_minutes_left=-1.0, dimming_window_relative_to_flare_minutes_right=1440.0, threshold_minimum_dimming_window_minutes=120.0, output_path='/Users/jmason86/Dropbox/Research/Postdoc_NASA/Analysis/Coronal Dimming Analysis/JEDI Catalog/', verbose=True): """Wrapper code for creating James's Extreme Ultraviolet Variability Experiment (EVE) Dimming Index (JEDI) catalog. Inputs: None. Optional Inputs: threshold_time_prior_flare_minutes [float]: How long before a particular event does the last one need to have occurred to be considered independent. If the previous one was too recent, will use that event's pre-flare irradiance. The mean dimming time of 100 dimming events in Reinard and Biesecker (2008, 2009) is the default. Default is 480 (8 hours). dimming_window_relative_to_flare_minutes_left [float]: Defines the left side of the time window to search for dimming relative to the GOES/XRS flare peak. Negative numbers mean minutes prior to the flare peak. Default is 0. dimming_window_relative_to_flare_minutes_right [float]: Defines the right side of the time window to search for dimming relative to the GOES/XRS flare peak. If another flare occurs before this, that time will define the end of the window instead. The time that "most" of the 100 dimming events had recovered by in Reinard and Biesecker (2008, 2009) is the default. Default is 1440 (24 hours). threshold_minimum_dimming_window_minutes [float]: The smallest allowed time window in which to search for dimming. Default is 120 (2 hours). flare_index_range [range] The range of GOES flare indices to process. Default is range(0, 5052). output_path [str]: Set to a path for saving the JEDI catalog table and processing summary plots. Default is '/Users/jmason86/Dropbox/Research/Postdoc_NASA/Analysis/Coronal Dimming Analysis/JEDI Catalog/'. verbose [bool]: Set to log the processing messages to disk and console. Default is False. Outputs: No direct return, but writes a csv to disk with the dimming paramerization results. Subroutines also optionally save processing plots to disk in output_path. Optional Outputs: None Example: generate_jedi_catalog(output_path='/Users/jmason86/Dropbox/Research/Postdoc_NASA/Analysis/Coronal Dimming Analysis/JEDI Catalog/', verbose=True) """ # Force flare_index_range to be an array type so it can be indexed in later code if isinstance(flare_index_range, int): flare_index_range = np.array([flare_index_range]) # Prepare the logger for verbose if verbose: logger = JpmLogger(filename='generate_jedi_catalog', path=output_path, console=False) logger.info("Starting JEDI processing pipeline.") logger.info("Processing events {0} - {1}".format( flare_index_range[0], flare_index_range[-1])) else: logger = None # Get EVE level 2 extracted emission lines data # TODO: Replace this shortcut method with the method I'm building into sunpy from scipy.io.idl import readsav eve_readsav = readsav( '/Users/jmason86/Dropbox/Research/Data/EVE/eve_lines_2010121-2014146 MEGS-A Mission Bare Bones.sav' ) if verbose: logger.info('Loaded EVE data') # Create metadata dictionary # TODO: Replace this shortcut method with the method I'm building into sunpy from sunpy.util.metadata import MetaDict metadata = MetaDict() metadata['ion'] = eve_readsav['name'] metadata['temperature_ion_peak_formation'] = np.power( 10.0, eve_readsav['logt']) * u.Kelvin metadata['extracted_wavelength_center'] = eve_readsav['wavelength'] * u.nm metadata['extracted_wavelength_min'] = metadata[ 'extracted_wavelength_center'] metadata['extracted_wavelength_max'] = metadata[ 'extracted_wavelength_center'] metadata['emission_line_blends'] = ['none', 'yay', 'poop', 'Fe vi'] # etc metadata[ 'exposure_time'] = 60.0 * u.second # These example EVE data are already binned down to 1 minute metadata['precision'] = ['Not implemented in prototype'] metadata['accuracy'] = ['Not implemented in prototype'] metadata['flags'] = ['Not implemented in prototype'] metadata['flags_description'] = '1 = MEGS-A data is missing, ' \ '2 = MEGS-B data is missing, ' \ '4 = ESP data is missing, ' \ '8 = MEGS-P data is missing, ' \ '16 = Possible clock adjust in MEGS-A, ' \ '32 = Possible clock adjust in MEGS-B, ' \ '64 = Possible clock adjust in ESP, ' \ '128 = Possible clock adjust in MEGS-P' metadata['flags_spacecraft'] = ['Not implemented in prototype'] metadata['flags_spacecraft_description'] = '0 = No obstruction, ' \ '1 = Warm up from Earth eclipse, ' \ '2 = Obstruction atmosphere penumbra, ' \ '3 = Obstruction atmosphere umbra, ' \ '4 = Obstruction penumbra of Mercury, ' \ '5 = Obstruction penumbra of Mercury, ' \ '6 = Obstruction penumbra of Venus, ' \ '7 = Obstruction umbra of Venus, ' \ '8 = Obstruction penumbra of Moon, ' \ '9 = Obstruction umbra of Moon, ' \ '10 = Obstruction penumbra of solid Earth, ' \ '11 = Obstruction umbra of solid Earth, ' \ '16 = Observatory is off-pointed by more than 1 arcmin' metadata['data_version'] = ['Not implemented in prototype'] metadata['data_reprocessed_revision'] = ['Not implemented in prototype'] metadata['filename'] = ['Not implemented in prototype'] # Load up the actual irradiance data into a pandas DataFrame # TODO: Replace this shortcut method with the method I'm building into sunpy irradiance = eve_readsav['irradiance'].byteswap().newbyteorder( ) # pandas doesn't like big endian irradiance[irradiance == -1] = np.nan wavelengths = eve_readsav['wavelength'] wavelengths_str = [] [ wavelengths_str.append('{0:1.1f}'.format(wavelength)) for wavelength in wavelengths ] eve_lines = pd.DataFrame(irradiance, columns=wavelengths_str) eve_lines.index = pd.to_datetime(eve_readsav.iso.astype(str)) eve_lines.sort_index(inplace=True) eve_lines = eve_lines.drop_duplicates() # Get GOES flare events above C1 within date range corresponding to EVE data # flares = get_goes_flare_events(eve_lines.index[0], eve_lines.index[-1], verbose=verbose) # TODO: The method in sunpy needs fixing, issue 2434 # Load GOES events from IDL saveset instead of directly through sunpy goes_flare_events = readsav( '/Users/jmason86/Dropbox/Research/Data/GOES/events/GoesEventsC1MinMegsAEra.sav' ) goes_flare_events['class'] = goes_flare_events['class'].astype(str) goes_flare_events['event_peak_time_human'] = goes_flare_events[ 'event_peak_time_human'].astype(str) goes_flare_events['event_start_time_human'] = goes_flare_events[ 'event_start_time_human'].astype(str) goes_flare_events['peak_time'] = Time( goes_flare_events['event_peak_time_jd'], format='jd', scale='utc') goes_flare_events['start_time'] = Time( goes_flare_events['event_start_time_jd'], format='jd', scale='utc') if verbose: logger.info('Loaded GOES flare events.') # Define the columns of the JEDI catalog jedi_row = pd.DataFrame([ OrderedDict([('Event #', np.nan), ('GOES Flare Start Time', np.nan), ('GOES Flare Peak Time', np.nan), ('GOES Flare Class', np.nan), ('Pre-Flare Start Time', np.nan), ('Pre-Flare End Time', np.nan), ('Flare Interrupt', np.nan)]) ]) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Pre-Flare Irradiance [W/m2]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Slope Start Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Slope End Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Slope Min [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Slope Max [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Slope Mean [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Slope Uncertainty [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Depth Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Depth [%]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Depth Uncertainty [%]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Duration Start Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Duration End Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Duration [s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Fitting Gamma')) jedi_row = jedi_row.join( pd.DataFrame(columns=eve_lines.columns + ' Fitting Score')) ion_tuples = list(itertools.permutations(eve_lines.columns.values, 2)) ion_permutations = pd.Index( [' by '.join(ion_tuples[i]) for i in range(len(ion_tuples))]) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Slope Start Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Slope End Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Slope Min [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Slope Max [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Slope Mean [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Slope Uncertainty [%/s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Depth Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Depth [%]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Depth Uncertainty [%]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Duration Start Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Duration End Time')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Duration [s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Correction Time Shift [s]')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Correction Scale Factor')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Fitting Gamma')) jedi_row = jedi_row.join( pd.DataFrame(columns=ion_permutations + ' Fitting Score')) csv_filename = output_path + 'jedi_{0}.csv'.format(Time.now().iso) jedi_row.to_csv(csv_filename, header=True, index=False, mode='w') if verbose: logger.info('Created JEDI row definition.') # Start a progress bar #widgets = [progressbar.Percentage(), progressbar.Bar(), progressbar.Timer(), ' ', progressbar.AdaptiveETA()] #progress_bar = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('Flare event loop: ')] + widgets, # min_value=flare_index_range[0], max_value=flare_index_range[-1]).start() # Prepare a hold-over pre-flare irradiance value, # which will normally have one element for each of the 39 emission lines preflare_irradiance = np.nan # Start loop through all flares for flare_index in flare_index_range: loop_time = time.time() # Skip event 0 to avoid problems with referring to earlier indices if flare_index == 0: continue print('Running on event {0}'.format(flare_index)) # Reset jedi_row jedi_row[:] = np.nan # Reset the flare interrupt flag flare_interrupt = False # Fill the GOES flare information into the JEDI row jedi_row['Event #'] = flare_index jedi_row['GOES Flare Start Time'] = goes_flare_events['start_time'][ flare_index].iso jedi_row['GOES Flare Peak Time'] = goes_flare_events['peak_time'][ flare_index].iso jedi_row['GOES Flare Class'] = goes_flare_events['class'][flare_index] if verbose: logger.info( "Event {0} GOES flare details stored to JEDI row.".format( flare_index)) # If haven't already done all pre-parameterization processing processed_jedi_non_params_filename = output_path + 'Processed Pre-Parameterization Data/Event {0} Pre-Parameterization.h5'.format( flare_index) processed_lines_filename = output_path + 'Processed Lines Data/Event {0} Lines.h5'.format( flare_index) if not os.path.isfile(processed_lines_filename) or not os.path.isfile( processed_jedi_non_params_filename): # Determine pre-flare irradiance minutes_since_last_flare = ( goes_flare_events['peak_time'][flare_index] - goes_flare_events['peak_time'][flare_index - 1]).sec / 60.0 if minutes_since_last_flare > threshold_time_prior_flare_minutes: # Clip EVE data from threshold_time_prior_flare_minutes prior to flare up to peak flare time preflare_window_start = ( goes_flare_events['peak_time'][flare_index] - (threshold_time_prior_flare_minutes * u.minute)).iso preflare_window_end = ( goes_flare_events['peak_time'][flare_index]).iso eve_lines_preflare_time = eve_lines[ preflare_window_start:preflare_window_end] # Loop through the emission lines and get pre-flare irradiance for each preflare_irradiance = [] for column in eve_lines_preflare_time: eve_line_preflare_time = pd.DataFrame( eve_lines_preflare_time[column]) eve_line_preflare_time.columns = ['irradiance'] preflare_irradiance.append( determine_preflare_irradiance( eve_line_preflare_time, pd.Timestamp(goes_flare_events['start_time'] [flare_index].iso), plot_path_filename=output_path + 'Preflare Determination/Event {0} {1}.png'.format( flare_index, column), verbose=verbose, logger=logger)) plt.close('all') else: logger.info( "This flare at {0} will use the pre-flare irradiance from flare at {1}." .format( goes_flare_events['peak_time'][flare_index].iso, goes_flare_events['peak_time'][flare_index - 1].iso)) # Make sure there is a preflare_window_start defined. If not, user probably needs to start from an earlier event. if 'preflare_window_start' not in locals(): print( 'ERROR: You need to start from an earlier event. Detected flare interrupt so need earlier pre-flare value.' ) logger.error( 'ERROR: You need to start from an earlier event. Detected flare interrupt so need earlier pre-flare value.' ) print('Moving to next event instead.') continue jedi_row["Pre-Flare Start Time"] = preflare_window_start jedi_row["Pre-Flare End Time"] = preflare_window_end preflare_irradiance_cols = [ col for col in jedi_row.columns if 'Pre-Flare Irradiance' in col ] jedi_row[preflare_irradiance_cols] = preflare_irradiance if verbose: logger.info( "Event {0} pre-flare determination complete.".format( flare_index)) # Clip EVE data to dimming window bracket_time_left = ( goes_flare_events['peak_time'][flare_index] + (dimming_window_relative_to_flare_minutes_left * u.minute)) next_flare_time = Time( (goes_flare_events['peak_time'][flare_index + 1]).iso) user_choice_time = ( goes_flare_events['peak_time'][flare_index] + (dimming_window_relative_to_flare_minutes_right * u.minute)) bracket_time_right = min(next_flare_time, user_choice_time) # If flare is shortening the window, set the flare_interrupt flag if bracket_time_right == next_flare_time: flare_interrupt = True if verbose: logger.info( 'Flare interrupt for event at {0} by flare at {1}'. format(goes_flare_events['peak_time'][flare_index].iso, next_flare_time)) # Write flare_interrupt to JEDI row jedi_row['Flare Interrupt'] = flare_interrupt # Skip event if the dimming window is too short if ((bracket_time_right - bracket_time_left).sec / 60.0) < threshold_minimum_dimming_window_minutes: # Leave all dimming parameters as NaN and write this null result to the CSV on disk jedi_row.to_csv(csv_filename, header=False, index=False, mode='a') # Log message if verbose: logger.info( 'The dimming window duration of {0} minutes is shorter than the minimum threshold of {1} minutes. Skipping this event ({2})' .format(((bracket_time_right - bracket_time_left).sec / 60.0), threshold_minimum_dimming_window_minutes, goes_flare_events['peak_time'][flare_index])) # Skip the rest of the processing in the flare_index loop continue else: eve_lines_event = eve_lines[bracket_time_left. iso:bracket_time_right.iso] if verbose: logger.info( "Event {0} EVE data clipped to dimming window.".format( flare_index)) # Convert irradiance units to percent # (in place, don't care about absolute units from this point forward) eve_lines_event = (eve_lines_event - preflare_irradiance ) / preflare_irradiance * 100.0 if verbose: logger.info( "Event {0} irradiance converted from absolute to percent units." .format(flare_index)) # Do flare removal in the light curves and add the results to the DataFrame #progress_bar_correction = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('Peak match subtract: ')] + widgets, # max_value=len(ion_tuples)).start() time_correction = time.time() for i in range(len(ion_tuples)): light_curve_to_subtract_from_df = pd.DataFrame( eve_lines_event[ion_tuples[i][0]]) light_curve_to_subtract_from_df.columns = ['irradiance'] light_curve_to_subtract_with_df = pd.DataFrame( eve_lines_event[ion_tuples[i][1]]) light_curve_to_subtract_with_df.columns = ['irradiance'] if (light_curve_to_subtract_from_df.isnull().all().all()) or ( light_curve_to_subtract_with_df.isnull().all().all()): if verbose: logger.info( 'Event {0} {1} correction skipped because all irradiances are NaN.' .format(flare_index, ion_permutations[i])) else: light_curve_corrected, seconds_shift, scale_factor = light_curve_peak_match_subtract( light_curve_to_subtract_from_df, light_curve_to_subtract_with_df, pd.Timestamp( (goes_flare_events['peak_time'][flare_index]).iso), plot_path_filename=output_path + 'Peak Subtractions/Event {0} {1}.png'.format( flare_index, ion_permutations[i]), verbose=verbose, logger=logger) eve_lines_event[ ion_permutations[i]] = light_curve_corrected jedi_row[ion_permutations[i] + ' Correction Time Shift [s]'] = seconds_shift jedi_row[ion_permutations[i] + ' Correction Scale Factor'] = scale_factor plt.close('all') if verbose: logger.info( 'Event {0} flare removal correction complete'. format(flare_index)) #progress_bar_correction.update(i) #progress_bar_correction.finish() print('Time to do peak match subtract [s]: {0}'.format( time.time() - time_correction)) # TODO: Update calculate_eve_fe_line_precision to compute for all emission lines, not just selected uncertainty = np.ones(len(eve_lines_event)) * 0.002545 # TODO: Propagate uncertainty through light_curve_peak_match_subtract and store in eve_lines_event # Fit the light curves to reduce influence of noise on the parameterizations to come later #progress_bar_fitting = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('Light curve fitting: ')] + widgets, # max_value=len(eve_lines_event.columns)).start() time_fitting = time.time() for i, column in enumerate(eve_lines_event): if eve_lines_event[column].isnull().all().all(): if verbose: logger.info( 'Event {0} {1} fitting skipped because all irradiances are NaN.' .format(flare_index, column)) else: eve_line_event = pd.DataFrame(eve_lines_event[column]) eve_line_event.columns = ['irradiance'] eve_line_event['uncertainty'] = uncertainty fitting_path = output_path + 'Fitting/' if not os.path.exists(fitting_path): os.makedirs(fitting_path) plt.close('all') light_curve_fit_df, best_fit_gamma, best_fit_score = light_curve_fit( eve_line_event, gamma=np.array([5e-8]), plots_save_path='{0}Event {1} {2} '.format( fitting_path, flare_index, column), verbose=verbose, logger=logger) eve_lines_event[column] = light_curve_fit_df jedi_row[column + ' Fitting Gamma'] = best_fit_gamma jedi_row[column + ' Fitting Score'] = best_fit_score if verbose: logger.info( 'Event {0} {1} light curves fitted.'.format( flare_index, column)) #progress_bar_fitting.update(i) #progress_bar_fitting.finish() print('Time to do fitting [s]: {0}'.format(time.time() - time_fitting)) # Save the dimming event data to disk for quicker restore jedi_row.to_hdf(processed_jedi_non_params_filename, 'jedi_row') eve_lines_event.to_hdf(processed_lines_filename, 'eve_lines_event') else: jedi_row = pd.read_hdf(processed_jedi_non_params_filename, 'jedi_row') eve_lines_event = pd.read_hdf(processed_lines_filename, 'eve_lines_event') if verbose: logger.info( 'Loading files {0} and {1} rather than processing again.'. format(processed_jedi_non_params_filename, processed_lines_filename)) # Parameterize the light curves for dimming for column in eve_lines_event: # Null out all parameters depth_percent, depth_time = np.nan, np.nan slope_start_time, slope_end_time = np.nan, np.nan slope_min, slope_max, slope_mean = np.nan, np.nan, np.nan duration_seconds, duration_start_time, duration_end_time = np.nan, np.nan, np.nan # Determine whether to do the parameterizations or not if eve_lines_event[column].isnull().all().all(): if verbose: logger.info( 'Event {0} {1} parameterization skipped because all irradiances are NaN.' .format(flare_index, column)) else: eve_line_event = pd.DataFrame(eve_lines_event[column]) eve_line_event.columns = ['irradiance'] # Determine dimming depth (if any) depth_path = output_path + 'Depth/' if not os.path.exists(depth_path): os.makedirs(depth_path) plt.close('all') depth_percent, depth_time = determine_dimming_depth( eve_line_event, plot_path_filename='{0}Event {1} {2} Depth.png'.format( depth_path, flare_index, column), verbose=verbose, logger=logger) jedi_row[column + ' Depth [%]'] = depth_percent # jedi_row[column + ' Depth Uncertainty [%]'] = depth_uncertainty # TODO: make determine_dimming_depth return the propagated uncertainty jedi_row[column + ' Depth Time'] = depth_time # Determine dimming slope (if any) slope_path = output_path + 'Slope/' if not os.path.exists(slope_path): os.makedirs(slope_path) slope_start_time = pd.Timestamp( (goes_flare_events['peak_time'][flare_index]).iso) slope_end_time = depth_time if (pd.isnull(slope_start_time)) or ( pd.isnull(slope_end_time)): if verbose: logger.warning( 'Cannot compute slope or duration because slope bounding times NaN.' ) else: plt.close('all') slope_min, slope_max, slope_mean = determine_dimming_slope( eve_line_event, earliest_allowed_time=slope_start_time, latest_allowed_time=slope_end_time, plot_path_filename='{0}Event {1} {2} Slope.png'.format( slope_path, flare_index, column), verbose=verbose, logger=logger) jedi_row[column + ' Slope Min [%/s]'] = slope_min jedi_row[column + ' Slope Max [%/s]'] = slope_max jedi_row[column + ' Slope Mean [%/s]'] = slope_mean # jedi_row[column + ' Slope Uncertainty [%]'] = slope_uncertainty # TODO: make determine_dimming_depth return the propagated uncertainty jedi_row[column + ' Slope Start Time'] = slope_start_time jedi_row[column + ' Slope End Time'] = slope_end_time # Determine dimming duration (if any) duration_path = output_path + 'Duration/' if not os.path.exists(duration_path): os.makedirs(duration_path) plt.close('all') duration_seconds, duration_start_time, duration_end_time = determine_dimming_duration( eve_line_event, earliest_allowed_time=slope_start_time, plot_path_filename='{0}Event {1} {2} Duration.png'. format(duration_path, flare_index, column), verbose=verbose, logger=logger) jedi_row[column + ' Duration [s]'] = duration_seconds jedi_row[column + ' Duration Start Time'] = duration_start_time jedi_row[column + ' Duration End Time'] = duration_end_time if verbose: logger.info( "Event {0} {1} parameterizations complete.".format( flare_index, column)) # Produce a summary plot for each light curve plt.style.use('jpm-transparent-light') ax = eve_line_event['irradiance'].plot(color='black') plt.axhline(linestyle='dashed', color='grey') start_date = eve_line_event.index.values[0] start_date_string = pd.to_datetime(str(start_date)) plt.xlabel(start_date_string.strftime('%Y-%m-%d %H:%M:%S')) plt.ylabel('Irradiance [%]') fmtr = dates.DateFormatter("%H:%M:%S") ax.xaxis.set_major_formatter(fmtr) ax.xaxis.set_major_locator(dates.HourLocator()) plt.title('Event {0} {1} nm Parameters'.format( flare_index, column)) if not np.isnan(depth_percent): plt.annotate('', xy=(depth_time, -depth_percent), xycoords='data', xytext=(depth_time, 0), textcoords='data', arrowprops=dict(facecolor='limegreen', edgecolor='limegreen', linewidth=2)) mid_depth = -depth_percent / 2.0 plt.annotate('{0:.2f} %'.format(depth_percent), xy=(depth_time, mid_depth), xycoords='data', ha='right', va='center', rotation=90, size=18, color='limegreen') if not np.isnan(slope_mean): p = plt.plot( eve_line_event[slope_start_time:slope_end_time] ['irradiance'], c='tomato') inverse_str = '$^{-1}$' plt.annotate('slope_min={0} % s{1}'.format( latex_float(slope_min), inverse_str), xy=(0.98, 0.12), xycoords='axes fraction', ha='right', size=12, color=p[0].get_color()) plt.annotate('slope_max={0} % s{1}'.format( latex_float(slope_max), inverse_str), xy=(0.98, 0.08), xycoords='axes fraction', ha='right', size=12, color=p[0].get_color()) plt.annotate('slope_mean={0} % s{1}'.format( latex_float(slope_mean), inverse_str), xy=(0.98, 0.04), xycoords='axes fraction', ha='right', size=12, color=p[0].get_color()) if not np.isnan(duration_seconds): plt.annotate('', xy=(duration_start_time, 0), xycoords='data', xytext=(duration_end_time, 0), textcoords='data', arrowprops=dict(facecolor='dodgerblue', edgecolor='dodgerblue', linewidth=5, arrowstyle='<->')) mid_time = duration_start_time + (duration_end_time - duration_start_time) / 2 plt.annotate(str(duration_seconds) + ' s', xy=(mid_time, 0), xycoords='data', ha='center', va='bottom', size=18, color='dodgerblue') summary_path = output_path + 'Summary Plots/' if not os.path.exists(summary_path): os.makedirs(summary_path) summary_filename = '{0}Event {1} {2} Parameter Summary.png'.format( summary_path, flare_index, column) plt.savefig(summary_filename) plt.close('all') if verbose: logger.info("Summary plot saved to %s" % summary_filename) # Write to the JEDI catalog on disk jedi_row.to_csv(csv_filename, header=False, index=False, mode='a') if verbose: logger.info('Event {0} JEDI row written to {1}.'.format( flare_index, csv_filename)) # Update progress bar #progress_bar.update(flare_index) print('Total time for loop [s]: {0}'.format(time.time() - loop_time))