def light_curve_peak_match_subtract(light_curve_to_subtract_from_df, light_curve_to_subtract_with_df, estimated_time_of_peak,
max_seconds_shift=1800,
plot_path_filename=None, verbose=False, logger=None):
"""Align the peak of a second light curve to the first, scale its magnitude to match, and subtract it off.
Inputs:
light_curve_to_subtract_from_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex and a column for irradiance.
light_curve_to_subtract_with_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex and a column for irradiance.
estimated_time_of_peak [metatime]: The estimated time that the peak should occur. This could come from, e.g., GOES/XRS.
Optional Inputs:
max_seconds_shift [int]: The maximum allowed time shift in seconds to get the peaks to match.
plot_path_filename [str]: Set to a path and filename in order to save the summary plot to disk.
Default is None, meaning the plot will not be saved to disk.
verbose [bool]: Set to log the processing messages to disk and console. Default is False.
logger [JpmLogger]: A configured logger from jpm_logger.py. If set to None, will generate a
new one. Default is None.
Outputs:
light_curve_corrected_df [pd DataFrame]: A pandas DataFrame with the same format as light_curve_to_subtract_from_df but
with the resultant peak match and subtraction performed. Returns np.nan if
the peaks couldn't be found.
seconds_shift [float]: The number of seconds that light_curve_to_subtract_with_df was shifted to get
its peak to match light_curve_to_subtract_from_df. Returns np.nan if
the peaks couldn't be found.
scale_factor [float]: The multiplicative factor applied to light_curve_to_subtract_with_df to get
its peak to match light_curve_to_subtract_from_df. Returns np.nan if
the peaks couldn't be found.
Optional Outputs:
None
Example:
light_curve_corrected_df, seconds_shift, scale_factor = light_curve_peak_match_subtract(light_curve_to_subtract_from_df,
light_curve_to_subtract_with_df,
estimated_time_of_peak,
plot_path_filename='./',
verbose=True)
"""
# Prepare the logger for verbose
if verbose:
if not logger:
logger = JpmLogger(filename='light_curve_peak_match_subtract_log', path='/Users/jmason86/Desktop/')
logger.info("Running on event with light curve start time of {0}.".format(light_curve_to_subtract_from_df.index[0]))
# Drop NaNs since peakutils can't handle them
light_curve_to_subtract_from_df = light_curve_to_subtract_from_df.dropna()
light_curve_to_subtract_with_df = light_curve_to_subtract_with_df.dropna()
# Detrend and find the peaks that are >= 95% of the max irradiance within
if verbose:
logger.info("Detrending light curves.")
if (light_curve_to_subtract_from_df['irradiance'].values < 0).all():
light_curve_to_subtract_from_df.iloc[0] = 1 # Else can crash peakutils.baseline
base_from = peakutils.baseline(light_curve_to_subtract_from_df)
detrend_from = light_curve_to_subtract_from_df - base_from
indices_from = peakutils.indexes(detrend_from.values.squeeze(), thres=0.95)
if (light_curve_to_subtract_with_df['irradiance'].values < 0).all():
light_curve_to_subtract_with_df.iloc[0] = 1 # Else can crash peakutils.baseline
base_with = peakutils.baseline(light_curve_to_subtract_with_df)
detrend_with = light_curve_to_subtract_with_df - base_with
indices_with = peakutils.indexes(detrend_with.values.squeeze(), thres=0.95)
if len(indices_from) == 0:
if verbose:
logger.warning('Could not find peak in light curve to subtract from.')
return np.nan, np.nan, np.nan
if len(indices_with) == 0:
if verbose:
logger.warning('Could not find peak in light curve to subtract with.')
return np.nan, np.nan, np.nan
# Identify the peak closest to the input estimated peak time (e.g., from GOES/XRS)
if verbose:
logger.info("Identifying peaks closest to initial guess in light curves.")
peak_index_from = indices_from[closest(light_curve_to_subtract_from_df.index[indices_from], estimated_time_of_peak)]
if len(indices_with) == 0:
import pdb
pdb.set_trace()
peak_index_with = indices_with[closest(light_curve_to_subtract_with_df.index[indices_with], estimated_time_of_peak)]
index_shift = peak_index_from - peak_index_with
# Compute how many seconds the time shift corresponds to
seconds_shift = (light_curve_to_subtract_from_df.index[peak_index_from] -
light_curve_to_subtract_with_df.index[peak_index_with]).total_seconds()
# Fail if seconds_shift > max_seconds_shift
isTimeShiftValid = True
if abs(seconds_shift) > max_seconds_shift:
if verbose:
logger.warning("Cannot do peak match. Time shift of {0} seconds is greater than max allowed shift of {1} seconds.".format(seconds_shift, max_seconds_shift))
isTimeShiftValid = False
# Shift the subtract_with light curve in time to align its peak to the subtract_from light curve
if isTimeShiftValid:
if verbose:
logger.info("Shifting and scaling the light curve to subtract with.")
shifted_with = light_curve_to_subtract_with_df.shift(index_shift)
# Scale the subtract_with light curve peak irradiance to match the subtract_from light curve peak irradiance
scale_factor = (detrend_from.values[peak_index_from] / shifted_with.values[peak_index_with + index_shift])[0]
shifted_scaled_with = shifted_with * scale_factor
light_curve_corrected_df = light_curve_to_subtract_from_df - shifted_scaled_with
if verbose:
if light_curve_corrected_df.isnull().values.sum() > 1:
logger.warning("%s points were shifted to become NaN." % light_curve_corrected_df.isnull().values.sum())
logger.info("Light curve peak matching and subtraction complete.")
if plot_path_filename:
from jpm_number_printing import latex_float
seconds_shift_string = '+' if seconds_shift >= 0 else ''
seconds_shift_string += str(int(seconds_shift))
if isTimeShiftValid:
scale_factor_string = latex_float(scale_factor)
plt.style.use('jpm-transparent-light')
from matplotlib import dates
plt.clf()
fig, ax = plt.subplots()
plt.plot(light_curve_to_subtract_from_df.index.values, light_curve_to_subtract_from_df.values, c='limegreen')
plt.tick_params(axis='x', which='minor', labelbottom='off')
plt.xlabel(estimated_time_of_peak)
plt.ylabel('Irradiance [%]')
fmtr = dates.DateFormatter("%H:%M:%S")
ax.xaxis.set_major_formatter(fmtr)
ax.xaxis.set_major_locator(dates.HourLocator())
if isTimeShiftValid:
plt.title('I: $\\times$' + scale_factor_string + ', t: ' + seconds_shift_string + ' s', color='tomato')
shifted_scaled_with.plot(c='tomato', label='subtract with', ax=ax)
light_curve_corrected_df.plot(c='darkgrey', label='result', ax=ax)
else:
plt.title('t: ' + seconds_shift_string + ' s > max allowed {0} s'.format(max_seconds_shift), color='tomato')
plt.plot(light_curve_to_subtract_with_df.index.values, light_curve_to_subtract_with_df.values, c='tomato')
plt.scatter(light_curve_to_subtract_from_df.index[peak_index_from], light_curve_to_subtract_from_df.values[peak_index_from], c='black')
if isTimeShiftValid:
plt.scatter(shifted_scaled_with.index[peak_index_with + index_shift], shifted_scaled_with.values[peak_index_with + index_shift], c='black')
ax.legend(['subtract from', 'subtract with', 'result'], loc='best')
else:
plt.scatter(light_curve_to_subtract_with_df.index[peak_index_with], light_curve_to_subtract_with_df.values[peak_index_with], c='black')
ax.legend(['subtract from', 'subtract with'], loc='best')
path = os.path.dirname(plot_path_filename)
if not os.path.exists(path):
os.makedirs(path)
plt.savefig(plot_path_filename)
if verbose:
logger.info("Summary plot saved to %s" % plot_path_filename)
if isTimeShiftValid:
return light_curve_corrected_df, seconds_shift, scale_factor
else:
return np.nan, seconds_shift, np.nan
def generate_cme_signature(start_timestamp='2010-08-07 17:12:11',
end_timestamp='2010-08-07 21:18:11',
output_path='/Users/tyleralbee/Desktop/StealthCME',
verbose=True):
"""Wrapper code for generating the dimming depth, duration, and slope for one CME event.
Inputs:
None.
Optional Inputs:
start_timestamp [str]: A timestamp for the beginning of an event.
end_timestamp [str]: A timestamp for the end of an event.
flare_index_range [range] The range of GOES flare indices to process. Default is range(0, 5052).
output_path [str]: Set to a path for saving the JEDI catalog table and processing
summary plots. Default is '/Users/shawnpolson/Documents/School/Spring 2018/Data Mining/StealthCMEs/PyCharm/JEDI Catalog/'.
verbose [bool]: Set to log the processing messages to disk and console. Default is False.
Outputs:
No direct return, but writes a (csv? sql table? hdf5?) to disk with the dimming paramerization results.
Subroutines also optionally save processing plots to disk in output_path.
Optional Outputs:
None
Example:
generate_cme_signature(verbose=True, start_timestamp='2010-08-07 17:12:11', end_timestamp='2010-08-07 23:18:11')
"""
# Prepare the logger for verbose
if verbose:
logger = JpmLogger(filename='generate_tylers_jedi_catalog',
path=output_path,
console=True)
logger.info("Starting CME signature processing pipeline.")
else:
logger = None
#---------Load dataset of selected lines--------------------------------------------------------------------------------
# Get EVE level 2 extracted emission lines data
# Load up the actual irradiance data into a pandas DataFrame
# Declare that column 0 is the index then convert it to datetime
eve_lines = pd.read_csv(
'/Users/tyleralbee/Desktop/StealthCME/eve_selected_lines.csv',
index_col=0)
eve_lines.index = pd.to_datetime(eve_lines.index)
if verbose:
logger.info('Loaded EVE data')
# Define the columns of the JEDI catalog
jedi_row = pd.DataFrame([
OrderedDict([('Event #', np.nan), ('Start Time', np.nan),
('End Time', np.nan), ('GOES Flare Class', np.nan),
('Pre-Flare Start Time', np.nan),
('Pre-Flare End Time', np.nan),
('Flare Interrupt', np.nan)])
])
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns +
' Pre-Flare Irradiance [W/m2]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Min [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Max [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Mean [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Uncertainty [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Depth Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Depth [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Depth Uncertainty [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Duration Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Duration End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Duration [s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Fitting Gamma'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Fitting Score'))
ion_tuples = list(itertools.permutations(eve_lines.columns.values, 2))
ion_permutations = pd.Index(
[' by '.join(ion_tuples[i]) for i in range(len(ion_tuples))])
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Min [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Max [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Mean [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Uncertainty [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Depth Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Depth [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Depth Uncertainty [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Duration Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Duration End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Duration [s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Correction Time Shift [s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Correction Scale Factor'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Fitting Gamma'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Fitting Score'))
csv_filename = output_path + 'jedi_{0}.csv'.format(Time.now().iso)
jedi_row.to_csv(csv_filename, header=True, index=False, mode='w')
if verbose:
logger.info('Created JEDI row definition.')
# Start a progress bar
widgets = [
progressbar.Percentage(),
progressbar.Bar(),
progressbar.Timer(), ' ',
progressbar.AdaptiveETA()
]
# Prepare a hold-over pre-flare irradiance value,
# which will normally have one element for each of the 39 emission lines
preflare_irradiance = np.nan
#----------Clip dataset to CME event window-----------------------------------------------------------------------------
# Get only rows in our dimming window
# Note: See this link if James's "eve_lines[start:end]" syntax is desired: https://stackoverflow.com/questions/16175874/python-pandas-dataframe-slicing-by-date-conditions (Note we get KeyError if requested times in this range don't exist exactly)
startTime = pd.to_datetime(
start_timestamp) # default value is '2010-08-07 17:12:11'
endTime = pd.to_datetime(
end_timestamp) # default value is '2010-08-07 21:18:11'
eve_lines_event = eve_lines.loc[(eve_lines.index >= startTime) & (
eve_lines.index <= endTime
)] # this syntax is more forgiving than "eve_lines[startTime:endTime]"
#print(eve_lines_event.head)
if verbose:
logger.info('Sliced rows in dimming window time range: ' +
start_timestamp + ' -> ' + end_timestamp)
logger.info("Event {0} EVE data clipped to dimming window.".format(1))
# Fill the event information into the JEDI row
jedi_row['Event #'] = 1
jedi_row['Start Time'] = start_timestamp
jedi_row['End Time'] = end_timestamp
if verbose:
logger.info("Event {0} details stored to JEDI row.".format(1))
#---------Convert irradiance values to percentages----------------------------------------------------------------------
# Convert irradiance units to percent
# (in place, don't care about absolute units from this point forward)
# Note: "preflare_irradiance" is pandas series with columns for each line and just one irradiance (float) per column
preflare_irradiance = eve_lines_event.iloc[0]
eve_lines_event_percentages = (
eve_lines_event - preflare_irradiance) / preflare_irradiance * 100.0
if verbose:
logger.info(
"Event {0} irradiance converted from absolute to percent units.".
format(1))
#---------Fit light curves to reduce noise------------------------------------------------------------------------------
# Fit the light curves to reduce influence of noise on the parameterizations to come later
uncertainty = np.ones(len(eve_lines_event_percentages)
) * 0.002545 # got this line from James's code
progress_bar_fitting = progressbar.ProgressBar(
widgets=[progressbar.FormatLabel('Light curve fitting: ')] + widgets,
max_value=len(eve_lines_event_percentages.columns)).start()
for i, column in enumerate(eve_lines_event_percentages):
if eve_lines_event_percentages[column].isnull().all().all():
if verbose:
logger.info(
'Event {0} {1} fitting skipped because all irradiances are NaN.'
.format(1, column))
else:
eve_line_event_percentages = pd.DataFrame(
eve_lines_event_percentages[column])
eve_line_event_percentages.columns = ['irradiance']
eve_line_event_percentages['uncertainty'] = uncertainty
fitting_path = output_path + 'Fitting/'
if not os.path.exists(fitting_path):
os.makedirs(fitting_path)
plt.close('all')
light_curve_fit, best_fit_gamma, best_fit_score = automatic_fit_light_curve(
eve_line_event_percentages,
plots_save_path='{0} Event {1} {2} '.format(
fitting_path, 1, column),
verbose=verbose,
logger=logger)
eve_lines_event_percentages[column] = light_curve_fit
jedi_row[column + ' Fitting Gamma'] = best_fit_gamma
jedi_row[column + ' Fitting Score'] = best_fit_score
if verbose:
logger.info('Event {0} {1} light curves fitted.'.format(
1, column))
progress_bar_fitting.update(i)
progress_bar_fitting.finish()
if verbose:
logger.info('Light curves fitted')
#---------Determine dimming characteristics-----------------------------------------------------------------------------
eve_lines_event = eve_lines_event_percentages # use the same variable name as James for consistency
# Parameterize the light curves for dimming
for column in eve_lines_event:
# Null out all parameters
depth_percent, depth_time = np.nan, np.nan
slope_start_time, slope_end_time = np.nan, np.nan
slope_min, slope_max, slope_mean = np.nan, np.nan, np.nan
duration_seconds, duration_start_time, duration_end_time = np.nan, np.nan, np.nan
# Determine whether to do the parameterizations or not
if eve_lines_event[column].isnull().all().all():
if verbose:
logger.info(
'Event {0} {1} parameterization skipped because all irradiances are NaN.'
.format(1, column))
else:
eve_line_event = pd.DataFrame(eve_lines_event[column])
eve_line_event.columns = ['irradiance']
# Determine dimming depth (if any)
depth_path = output_path + 'Depth/'
if not os.path.exists(depth_path):
os.makedirs(depth_path)
plt.close('all')
# Call to "determine_dimming_depth()"
depth_percent, depth_time = determine_dimming_depth(
eve_line_event,
plot_path_filename='{0} Event {1} {2} Depth.png'.format(
depth_path, 1, column),
verbose=verbose,
logger=logger)
jedi_row[column + ' Depth [%]'] = depth_percent
jedi_row[column + ' Depth Time'] = depth_time
# Determine dimming slope (if any)
slope_path = output_path + 'Slope/'
if not os.path.exists(slope_path):
os.makedirs(slope_path)
slope_start_time = startTime # Note: this is the same startTime as line 141 ("start_timestamp" from main)
slope_end_time = depth_time
if (pd.isnull(slope_start_time)) or (pd.isnull(slope_end_time)):
if verbose:
logger.warning(
'Cannot compute slope or duration because slope bounding times NaN.'
)
else:
plt.close('all')
# Call to "determine_dimming_slope()"
slope_min, slope_max, slope_mean = determine_dimming_slope(
eve_line_event,
earliest_allowed_time=slope_start_time,
latest_allowed_time=slope_end_time,
plot_path_filename='{0} Event {1} {2} Slope.png'.format(
slope_path, 1, column),
verbose=verbose,
logger=logger)
jedi_row[column + ' Slope Min [%/s]'] = slope_min
jedi_row[column + ' Slope Max [%/s]'] = slope_max
jedi_row[column + ' Slope Mean [%/s]'] = slope_mean
jedi_row[column + ' Slope Start Time'] = slope_start_time
jedi_row[column + ' Slope End Time'] = slope_end_time
# Determine dimming duration (if any)
duration_path = output_path + 'Duration/'
if not os.path.exists(duration_path):
os.makedirs(duration_path)
plt.close('all')
# Call to "determine_dimming_duration()"
duration_seconds, duration_start_time, duration_end_time = determine_dimming_duration(
eve_line_event,
earliest_allowed_time=slope_start_time,
plot_path_filename='{0} Event {1} {2} Duration.png'.format(
duration_path, 1, column),
verbose=verbose,
logger=logger)
jedi_row[column + ' Duration [s]'] = duration_seconds
jedi_row[column + ' Duration Start Time'] = duration_start_time
jedi_row[column + ' Duration End Time'] = duration_end_time
if verbose:
logger.info("Event {0} {1} parameterizations complete.".format(
1, column))
#----------Plot everything----------------------------------------------------------------------------------
# Produce a summary plot for each light curve
plt.style.use('jpm-transparent-light')
ax = eve_line_event['irradiance'].plot(color='black')
plt.axhline(linestyle='dashed', color='grey')
start_date = eve_line_event.index.values[0]
start_date_string = pd.to_datetime(str(start_date))
plt.xlabel(start_date_string.strftime('%Y-%m-%d %H:%M:%S'))
plt.ylabel('Irradiance [%]')
fmtr = dates.DateFormatter("%H:%M:%S")
ax.xaxis.set_major_formatter(fmtr)
ax.xaxis.set_major_locator(dates.HourLocator())
plt.title('Event {0} {1} nm Parameters'.format(1, column))
if not np.isnan(depth_percent):
plt.annotate('',
xy=(depth_time, -depth_percent),
xycoords='data',
xytext=(depth_time, 0),
textcoords='data',
arrowprops=dict(facecolor='limegreen',
edgecolor='limegreen',
linewidth=2))
mid_depth = -depth_percent / 2.0
plt.annotate('{0:.2f} %'.format(depth_percent),
xy=(depth_time, mid_depth),
xycoords='data',
ha='right',
va='center',
rotation=90,
size=18,
color='limegreen')
if not np.isnan(slope_mean):
if pd.isnull(slope_start_time) or pd.isnull(slope_end_time):
import pdb
pdb.set_trace()
p = plt.plot(eve_line_event[slope_start_time:slope_end_time]
['irradiance'],
c='tomato')
inverse_str = '$^{-1}$'
plt.annotate('slope_min={0} % s{1}'.format(
latex_float(slope_min), inverse_str),
xy=(0.98, 0.12),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
plt.annotate('slope_max={0} % s{1}'.format(
latex_float(slope_max), inverse_str),
xy=(0.98, 0.08),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
plt.annotate('slope_mean={0} % s{1}'.format(
latex_float(slope_mean), inverse_str),
xy=(0.98, 0.04),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
if not np.isnan(duration_seconds):
plt.annotate('',
xy=(duration_start_time, 0),
xycoords='data',
xytext=(duration_end_time, 0),
textcoords='data',
arrowprops=dict(facecolor='dodgerblue',
edgecolor='dodgerblue',
linewidth=5,
arrowstyle='<->'))
mid_time = duration_start_time + (duration_end_time -
duration_start_time) / 2
plt.annotate(str(duration_seconds) + ' s',
xy=(mid_time, 0),
xycoords='data',
ha='center',
va='bottom',
size=18,
color='dodgerblue')
summary_path = output_path + 'Summary Plots/'
if not os.path.exists(summary_path):
os.makedirs(summary_path)
summary_filename = '{0} Event {1} {2} Parameter Summary.png'.format(
summary_path, 1, column)
plt.savefig(summary_filename)
if verbose:
logger.info("Summary plot saved to %s" % summary_filename)
#---------Output results------------------------------------------------------------------------------------------------
# Write to the JEDI catalog on disk
jedi_row.to_csv(csv_filename, header=False, index=False, mode='a')
if verbose:
logger.info('Event {0} JEDI row written to {1}.'.format(
1, csv_filename))
def automatic_fit_light_curve(light_curve_df, minimum_score=0.3, plots_save_path=None,
verbose=False, logger=None):
"""Automatically fit the best support vector machine regression (SVR) model for the input light curve.
Inputs:
light_curve_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex, and columns for irradiance and uncertainty.
Optional Inputs:
minimum_score [float]: Set this to the minimum explained variance score (0 - 1) acceptable for fits. If the
best fit score is < minimum_score, this function will return np.nan for light_curve_fit.
Default value is 0.3.
plots_save_path [str]: Set to a path in order to save the validation curve and best fit overplot on the data to disk.
Default is None, meaning no plots will be saved to disk.
verbose [bool]: Set to log the processing messages to disk and console. Default is False.
logger [JpmLogger]: A configured logger from jpm_logger.py. If set to None, will generate a
new one. Default is None.
Outputs:
light_curve_fit_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex, and columns for fitted irradiance and uncertainty.
best_fit_gamma [float]: The best found gamma hyper parameter for the SVR.
best_fit_score [float]: The best explained variance score.
Optional Outputs:
None
Example:
light_curve_fit, best_fit_gamma, best_fit_score = automatic_fit_light_curve(light_curve_df, verbose=True)
"""
# Prepare the logger for verbose
if verbose:
if not logger:
logger = JpmLogger(filename='automatic_fit_light_curve_log', path='/Users/jmason86/Desktop/')
logger.info("Running on event with light curve start time of {0}.".format(light_curve_df.index[0]))
# Pull data out of the DataFrame for compatibility formatting
X = metatimes_to_seconds_since_start(light_curve_df.index)
y = light_curve_df['irradiance'].values
# Check for NaNs and issue warning that they are being removed from the dataset
if verbose:
if np.isnan(y).any():
logger.warning("There are NaN values in light curve. Dropping them.")
finite_irradiance_indices = np.isfinite(y)
X = X[finite_irradiance_indices]
X = X.reshape(len(X), 1) # Format to be compatible with validation_curve and SVR.fit()
uncertainty = light_curve_df.uncertainty[np.isfinite(y)]
y = y[finite_irradiance_indices]
if verbose:
logger.info("Fitting %s points." % len(y))
# Helper function for compatibility with validation_curve
def jpm_svr(gamma=1e-6, **kwargs):
return make_pipeline(SVR(kernel='rbf', C=1e3, gamma=gamma, **kwargs))
# Hyper parameter for SVR is gamma, so generate values of it to try
gamma = np.logspace(-7, 1, num=20, base=10)
# Overwrite the default scorer (R^2) with explained variance score
evs = make_scorer(explained_variance_score)
# Split the data between training/testing 50/50 but across the whole time range rather than the default consecutive Kfolds
import time
t0 = time.time()
shuffle_split = ShuffleSplit(n_splits=20, train_size=0.5, test_size=0.5, random_state=None)
# Generate the validation curve -- test all them gammas!
# Parallelized to speed it up (n_jobs = # of parallel threads)
train_score, val_score = validation_curve(jpm_svr(), X, y,
'svr__gamma',
gamma, cv=shuffle_split, n_jobs=7, scoring=evs)
t1 = time.time()
logger.error('It took {0} seconds to run.'.format(t1 - t0))
if verbose:
logger.info("Validation curve complete.")
if plots_save_path:
plt.clf()
plt.style.use('jpm-transparent-light')
plt.plot(gamma, np.median(train_score, 1), label='training score')
plt.plot(gamma, np.median(val_score, 1), label='validation score')
ax = plt.axes()
plt.legend(loc='best')
plt.title("t$_0$ = " + datetimeindex_to_human(light_curve_df.index)[0])
ax.set_xscale('log')
plt.xlabel('gamma')
plt.ylabel('score')
plt.ylim(0, 1)
filename = plots_save_path + 'Validation Curve t0 ' + datetimeindex_to_human(light_curve_df.index)[0] + '.png'
plt.savefig(filename)
if verbose:
logger.info("Validation curve saved to %s" % filename)
# Identify the best score
scores = np.median(val_score, axis=1)
best_fit_score = np.max(scores)
best_fit_gamma = gamma[np.argmax(scores)]
if verbose:
logger.info('Scores: ' + str(scores))
logger.info('Best score: ' + str(best_fit_score))
logger.info('Best fit gamma: ' + str(best_fit_gamma))
# Return np.nan if only got bad fits
if best_fit_score < minimum_score:
if verbose:
logger.warning("Uh oh. Best fit score {0:.2f} is < user-defined minimum score {1:.2f}".format(best_fit_score, minimum_score))
return np.nan, best_fit_gamma, best_fit_score
# Otherwise train and fit the best model
sample_weight = 1 / uncertainty
model = SVR(kernel='rbf', C=1e3, gamma=best_fit_gamma).fit(X, y, sample_weight)
y_fit = model.predict(X)
if verbose:
logger.info("Best model trained and fitted.")
if plots_save_path:
plt.clf()
plt.errorbar(X.ravel(), y, yerr=uncertainty, color='black', fmt='o', label='Input light curve')
plt.plot(X.ravel(), y_fit, linewidth=6, label='Fit')
plt.title("t$_0$ = " + datetimeindex_to_human(light_curve_df.index)[0])
plt.xlabel('time [seconds since start]')
plt.ylabel('irradiance [%]')
plt.legend(loc='best')
filename = plots_save_path + 'Fit t0 ' + datetimeindex_to_human(light_curve_df.index)[0] + '.png'
plt.savefig(filename)
if verbose:
logger.info("Fitted curve saved to %s" % filename)
# TODO: Get uncertainty of fit at each point... if that's even possible
# Placeholder for now just so that the function can complete: output uncertainty = input uncertainty
fit_uncertainty = uncertainty
# Construct a pandas DataFrame with DatetimeIndex, y_fit, and fit_uncertainty
light_curve_fit_df = pd.DataFrame({'irradiance': y_fit,
'uncertainty': fit_uncertainty})
light_curve_fit_df.index = light_curve_df.index[finite_irradiance_indices]
if verbose:
logger.info("Created output DataFrame")
return light_curve_fit_df, best_fit_gamma, best_fit_score
def determine_dimming_slope(light_curve_df,
earliest_allowed_time=None,
latest_allowed_time=None,
smooth_points=0,
plot_path_filename=None,
verbose=False,
logger=None):
"""Find the slope of dimming in a light curve, if any.
Inputs:
light_curve_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex and a column for irradiance.
Optional Inputs:
earliest_allowed_time [metatime]: The function won't return a slope determined any earlier than this.
It is recommended that this be the peak time of the flare.
Default is None, meaning the beginning of the light_curve_df.
latest_allowed_time [metatime]: The function won't return a slope determined any later than this.
It is recommended that this be the identified time of dimming depth.
Default is None, meaning the end of the light_curve_df.
smooth_points [integer]: Used to apply a rolling mean with the number of points (indices) specified.
Default is 0, meaning no smoothing will be performed.
plot_path_filename [str]: Set to a path and filename in order to save the summary plot to disk.
Default is None, meaning the plot will not be saved to disk.
verbose [bool]: Set to log the processing messages to disk and console. Default is False.
logger [JpmLogger]: A configured logger from jpm_logger.py. If set to None, will generate a
new one. Default is None.
Outputs:
slope_min [float]: The minimum slope of dimming in percent/second terms.
slope_max [float]: The maximum slope of dimming in percent/second terms.
slope_mean [float]: The mean slope of dimming in percent/second terms.
Optional Outputs:
None
Example:
slope_min, slope_max, slope_mean = determine_dimming_slope(light_curve_df,
plot_path_filename='./determine_dimming_slope_summary.png',
verbose=True)
"""
# Prepare the logger for verbose
if verbose:
if not logger:
logger = JpmLogger(filename='determine_dimming_slope_log',
path='/Users/jmason86/Desktop/')
logger.info(
"Running on event with light curve start time of {0}.".format(
light_curve_df.index[0]))
# If no earliest_allowed_time set, then set it to beginning of light_curve_df
if not earliest_allowed_time:
earliest_allowed_time = light_curve_df.index[0]
logger.info(
"No earliest allowed time provided. Setting to beginning of light curve: {0}"
.format(earliest_allowed_time))
# If no latest_allowed_time set, then set it to end of light_curve_df
if not latest_allowed_time:
latest_allowed_time = light_curve_df.index[-1]
logger.info(
"No latest allowed time provided. Setting to end of light curve: {0}"
.format(latest_allowed_time))
# Optionally smooth the light curve with a rolling mean
if smooth_points:
light_curve_df['irradiance'] = light_curve_df.rolling(
smooth_points, center=True).mean()
if verbose:
logger.info('Applied {0} point smooth.'.format(smooth_points))
first_non_nan = light_curve_df['irradiance'].first_valid_index()
nan_indices = np.isnan(light_curve_df['irradiance'])
light_curve_df['irradiance'][nan_indices] = light_curve_df['irradiance'][
first_non_nan]
# Find the max in the allowed window
max_time = light_curve_df[earliest_allowed_time:latest_allowed_time][
'irradiance'].idxmax()
max_irradiance = light_curve_df['irradiance'].loc[max_time]
if verbose:
logger.info(
'Maximum in allowed window found with value of {0:.2f} at time {1}'
.format(max_irradiance, max_time))
# Compute the derivative in the time window of interest (inverting sign so that we describe "downward slope")
derivative = -light_curve_df[max_time:latest_allowed_time][
'irradiance'].diff(
) / light_curve_df[max_time:latest_allowed_time].index.to_series(
).diff().dt.total_seconds()
if verbose:
logger.info(
"Computed derivative of light curve within time window of interest."
)
# Get the min, max, and mean slope
slope_min = derivative.min()
slope_max = derivative.max()
slope_mean = derivative.mean()
slope_min_str = latex_float(slope_min)
slope_max_str = latex_float(slope_max)
slope_mean_str = latex_float(slope_mean)
if verbose:
logger.info(
"Computed min ({0}), max ({1}), and mean ({2}) %/s slope.".format(
slope_min_str, slope_max_str, slope_mean_str))
# Do a few sanity checks for the log
if verbose:
if slope_min < 0:
logger.warning(
"Minimum slope of {0} is unexpectedly < 0.".format(slope_min))
if slope_max < 0:
logger.warning(
"Maximum slope of {0} is unexpectedly < 0.".format(slope_max))
if slope_mean < 0:
logger.warning(
"Mean slope of {0} is unexpectedly < 0.".format(slope_mean))
# Produce a summary plot
if plot_path_filename:
plt.style.use('jpm-transparent-light')
from matplotlib import dates
p = plt.plot(light_curve_df['irradiance'])
p = plt.plot(
light_curve_df[max_time:latest_allowed_time]['irradiance'],
label='slope region')
ax = plt.gca()
plt.axvline(x=earliest_allowed_time, linestyle='dashed', color='grey')
plt.axvline(x=latest_allowed_time, linestyle='dashed', color='black')
plt.axvline(x=max_time, linestyle='dashed', color='black')
plt.title('Identified Slope')
start_date = light_curve_df.index.values[0]
start_date_string = pd.to_datetime(str(start_date))
plt.xlabel(start_date_string.strftime('%Y-%m-%d %H:%M:%S'))
fmtr = dates.DateFormatter("%H:%M:%S")
ax.xaxis.set_major_formatter(fmtr)
ax.xaxis.set_major_locator(dates.HourLocator())
ax.xaxis.grid(b=True, which='minor')
plt.ylabel('Irradiance [%]')
inverse_str = '$^{-1}$'
plt.annotate('slope_min={0} % sec{1}'.format(slope_min_str,
inverse_str),
xy=(0.98, 0.12),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
plt.annotate('slope_max={0} % sec{1}'.format(slope_max_str,
inverse_str),
xy=(0.98, 0.08),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
plt.annotate('slope_mean={0} % sec{1}'.format(slope_mean_str,
inverse_str),
xy=(0.98, 0.04),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
ax.legend(loc='best')
plt.savefig(plot_path_filename)
if verbose:
logger.info("Summary plot saved to %s" % plot_path_filename)
# Return the slopes
return slope_min, slope_max, slope_mean
def determine_dimming_duration(light_curve_df,
earliest_allowed_time=None, smooth_points=0,
plot_path_filename=None, verbose=False, logger=None):
"""Find the duration of dimming in a light curve, if any.
Assumes light curve is normalized such that pre-flare = 0%.
Inputs:
light_curve_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex and a column for irradiance.
Optional Inputs:
earliest_allowed_time [metatime]: The function won't return a duration if the only 0 crossings are earlier than this.
Default is None, meaning the beginning of the light_curve_df.
smooth_points [integer]: Used to apply a rolling mean with the number of points (indices) specified.
Default is 0, meaning no smoothing will be performed.
plot_path_filename [str]: Set to a path and filename in order to save the summary plot to disk.
Default is None, meaning the plot will not be saved to disk.
verbose [bool]: Set to log the processing messages to disk and console. Default is False.
logger [JpmLogger]: A configured logger from jpm_logger.py. If set to None, will generate a
new one. Default is None.
Outputs:
duration_seconds [integer]: The duration of dimming in seconds.
duration_start_time [pd.Timestamp]: The time the duration starts (downward 0 crossing).
duration_end_time [pd.Timestamp]: The time the duration ends (upward 0 crossing).
Optional Outputs:
None
Example:
duration_seconds, duration_start_time, duration_end_time = determine_dimming_duration(light_curve_df,
plot_path_filename='./bla.png',
verbose=True)
"""
# If no earliest_allowed_time set, then set it to beginning of light_curve_df
if not earliest_allowed_time:
earliest_allowed_time = pd.Timestamp(light_curve_df.index.values[0])
# Prepare the logger for verbose
if verbose:
if not logger:
logger = JpmLogger(filename='determine_dimming_duration_log', path='/Users/jmason86/Desktop/')
logger.info("Running on event with light curve start time of {0}.".format(light_curve_df.index[0]))
# Set up a successful processing flag
found_duration = True
# Optionally smooth the light curve with a rolling mean
if smooth_points:
light_curve_df['smooth'] = light_curve_df.rolling(smooth_points, center=True).mean()
else:
light_curve_df['smooth'] = light_curve_df['irradiance']
first_non_nan = light_curve_df['smooth'].first_valid_index()
nan_indices = np.isnan(light_curve_df['smooth'])
light_curve_df['smooth'][nan_indices] = light_curve_df['smooth'][first_non_nan]
# Find the indices where the light curve is closest to 0
zero_crossing_indices = np.where(np.diff(np.signbit(light_curve_df['smooth'])))[0]
zero_crossing_times = light_curve_df.index[zero_crossing_indices]
# Discard any indices prior to the user-provided earliest_allowed_time, else cannot compute
zero_crossing_indices = zero_crossing_indices[zero_crossing_times > earliest_allowed_time]
if zero_crossing_indices.size == 0:
if verbose:
logger.warning('No zero crossings detected after earliest allowed time of %s' % earliest_allowed_time)
found_duration = False
# Figure out which way the light curve is sloping
if found_duration:
light_curve_df['diff'] = light_curve_df['smooth'].diff()
# Find the first negative slope zero crossing time
if found_duration:
neg_zero_crossing_indices = np.where(light_curve_df['diff'][zero_crossing_indices + 1] < 0)[0]
if len(neg_zero_crossing_indices) > 0:
first_neg_zero_crossing_index = neg_zero_crossing_indices[0]
first_neg_zero_crossing_time = light_curve_df.index[zero_crossing_indices[first_neg_zero_crossing_index]]
else:
if verbose:
logger.warning('No negative slope 0-crossing found. Duration cannot be defined.')
found_duration = False
# Find the first postiive slope zero crossing
if found_duration:
pos_zero_crossing_indices = np.where(light_curve_df['diff'][zero_crossing_indices + 1] > 0)[0]
if len(pos_zero_crossing_indices) > 0:
first_pos_zero_crossing_index = pos_zero_crossing_indices[0]
first_pos_zero_crossing_time = light_curve_df.index[zero_crossing_indices[first_pos_zero_crossing_index]]
else:
if verbose:
logger.warning('No positive slope 0-crossing found. Duration cannot be defined.')
found_duration = False
# If the first negative slope zero crossing isn't earlier than the positive one, return null
if (found_duration) and (first_neg_zero_crossing_time > first_pos_zero_crossing_time):
if verbose:
logger.warning('Dimming light curve may be misaligned in window. Negative slope 0-crossing detected after positive one.')
found_duration = False
# Return the time difference in seconds between the selected zero crossings
if found_duration:
duration_seconds = int((first_pos_zero_crossing_time - first_neg_zero_crossing_time).total_seconds())
if plot_path_filename:
plt.style.use('jpm-transparent-light')
from matplotlib import dates
if found_duration:
light_curve_df = light_curve_df.drop('diff', 1)
ax = light_curve_df['irradiance'].plot()
start_date = light_curve_df.index.values[0]
start_date_string = pd.to_datetime(str(start_date))
plt.xlabel(start_date_string.strftime('%Y-%m-%d %H:%M:%S'))
plt.ylabel('Irradiance [%]')
fmtr = dates.DateFormatter("%H:%M:%S")
ax.xaxis.set_major_formatter(fmtr)
ax.xaxis.set_major_locator(dates.HourLocator())
plt.title('Dimming Duration')
if found_duration:
plt.scatter([zero_crossing_times[first_neg_zero_crossing_index], zero_crossing_times[first_pos_zero_crossing_index]],
[light_curve_df['smooth'][zero_crossing_indices[first_neg_zero_crossing_index]],
light_curve_df['smooth'][zero_crossing_indices[first_pos_zero_crossing_index]]],
c='black', s=300, zorder=3)
plt.annotate('', xy=(first_neg_zero_crossing_time, 0), xycoords='data',
xytext=(first_pos_zero_crossing_time, 0), textcoords='data',
arrowprops=dict(facecolor='black', linewidth=5, arrowstyle='<->'))
mid_time = first_neg_zero_crossing_time + (first_pos_zero_crossing_time - first_neg_zero_crossing_time) / 2
plt.annotate(str(duration_seconds) + ' s', xy=(mid_time, 0), xycoords='data', ha='center', va='bottom', size=18)
plt.savefig(plot_path_filename)
if verbose:
logger.info("Summary plot saved to %s" % plot_path_filename)
if not found_duration:
duration_seconds = np.nan
first_neg_zero_crossing_time = np.nan
first_pos_zero_crossing_time = np.nan
return duration_seconds, first_neg_zero_crossing_time, first_pos_zero_crossing_time
def determine_preflare_irradiance(light_curve_df,
estimated_time_of_peak_start,
max_median_diff_threshold=1.5,
std_threshold=1.0,
plot_path_filename=None,
verbose=False,
logger=None):
"""Determine pre-flare irradiance level in a solar light curve.
Or, more generally, find the pre-peak level in a time series.
Inputs:
light_curve_df [pd DataFrame]: A pandas DataFrame with a DatetimeIndex and a column for irradiance.
estimated_time_of_peak_start [metatime]: The estimated time that the dramatic increase starts.
This could come from, e.g., GOES/XRS.
Optional Inputs:
max_median_diff_threshold [float]: The maximum allowed difference in medians between the 3 pre-flare windows
in percent terms. This value gets multiplied by the mean of the stds from
each sub-window and is then compared to the max_median_diff. The default is 1.5.
std_threshold [float]: The maximum allowed standard deviation in the pre-flare windows in percent
terms. The default is 0.5.
plot_path_filename [str]: Set to a path and filename in order to save the summary plot to disk.
Default is None, meaning the plot will not be saved to disk.
verbose [bool]: Set to log the processing messages to disk and console. Default is False.
logger [JpmLogger]: A configured logger from jpm_logger.py. If set to None, will generate a
new one. Default is None.
Outputs:
preflare_irradiance [float]: The identified pre-flare irradiance level in the same units as light_curve_df.irradiance.
Optional Outputs:
None
Example:
preflare_irradiance = determine_preflare_irradiance(light_curve_df, pd.Timestamp('2012-04-15 17:52:20.0'),
plot_path_filename='./bla.png',
verbose=True)
"""
# Prepare the logger for verbose
if verbose:
if not logger:
logger = JpmLogger(filename='determine_preflare_irradiance_log',
path='/Users/jmason86/Desktop/')
logger.info("Running on event with peak start time of {0}.".format(
estimated_time_of_peak_start))
# Verify that not all values are nan
if light_curve_df.isna().all().all():
if verbose:
logger.warning("All irradiance values are NaN. Returning.")
return np.nan
# Convert irradiance to percent if not already present
if 'irradiance_percent' not in light_curve_df.columns:
median_irradiance = light_curve_df['irradiance'].median()
light_curve_df['irradiance_percent'] = (
light_curve_df['irradiance'].values -
median_irradiance) / median_irradiance * 100.
if verbose:
logger.info(
"Converted irradiance to percent, baselining median in entire pre-flare window."
)
# Divide the pre-flare period into 3 equal-length windows
windows = np.array_split(light_curve_df[:estimated_time_of_peak_start], 3)
if verbose:
logger.info("Divided pre-flare period into 3 equal-length windows.")
# Compute median and standard deviation in each window
medians = [
windowed_df['irradiance_percent'].median() for windowed_df in windows
]
medians_abs = [
windowed_df['irradiance'].median() for windowed_df in windows
]
stds = np.array(
[windowed_df['irradiance_percent'].std() for windowed_df in windows])
if verbose:
logger.info("Computed medians and standard deviations in each window.")
# Compute max difference between the medians
max_median_diff = np.max(
np.abs(np.append(np.diff(medians), medians[2] - medians[0])))
# Compare medians and standard deviations in each window to thresholds
failed_median_threshold = False
failed_std_threshold = False
if np.all(np.isnan(stds)):
if verbose:
logger.warning(
'Cannot compute pre-flare irradiance. All standard deviations are nan.'
)
failed_std_threshold = True
else:
if max_median_diff > max_median_diff_threshold * np.mean(stds):
if verbose:
logger.warning(
'Cannot compute pre-flare irradiance. Maximum difference in window medians ({0}) exceeded threshold ({1}).'
.format(max_median_diff,
max_median_diff_threshold * np.mean(stds)))
failed_median_threshold = True
if (stds < std_threshold).sum() < 2:
if verbose:
logger.warning(
'Cannot compute pre-flare irradiance. Standard deviation in more than 1 window is larger than threshold ({0}).'
.format(std_threshold))
failed_std_threshold = True
# Compute pre-flare irradiance (mean of the medians in absolute units)
if failed_median_threshold or failed_std_threshold:
preflare_irradiance = np.nan
else:
preflare_irradiance = np.mean(
[windowed_df['irradiance'].median() for windowed_df in windows])
if verbose:
logger.info("Computed pre-flare irradiance: {0}".format(
preflare_irradiance))
# Produce summary plot
if plot_path_filename:
plt.style.use('jpm-transparent-light')
from matplotlib import dates
from matplotlib.patches import Rectangle
light_curve_df = light_curve_df.drop('irradiance_percent', 1)
ax = light_curve_df[:estimated_time_of_peak_start].plot(legend=False,
c='grey')
# plt.plot(light_curve_df[:estimated_time_of_peak_start].irradiance, c='grey') # using matplotlib instead of pandas
# ax = plt.gca()
start_date = light_curve_df.index.values[0]
start_date_string = pd.to_datetime(str(start_date))
plt.title('Pre-flare Windows')
plt.xlabel(start_date_string.strftime('%Y-%m-%d %H:%M:%S'))
plt.ylabel('Irradiance [W m$^{-2}$]')
fmtr = dates.DateFormatter("%H:%M:%S")
ax.xaxis.set_major_formatter(fmtr)
ax.xaxis.set_major_locator(dates.HourLocator())
ax2 = ax.twinx()
light_curve_df[:estimated_time_of_peak_start].plot(ax=ax2,
legend=False,
c='grey')
# ax2.plot(light_curve_df[:estimated_time_of_peak_start].irradiance, color='grey')
vals = ax2.get_yticks()
ax2.set_yticklabels([
'{:3.2f}%'.format(
(x - median_irradiance) / median_irradiance * 100)
for x in vals
])
# First window
start = dates.date2num(light_curve_df.index[0])
end = dates.date2num(windows[0].index[-1])
width = end - start
rect = Rectangle((start, 0), width, 1, color='deepskyblue', alpha=0.2)
ax.add_patch(rect)
plt.plot([windows[0].index[0], windows[0].index[-1]],
[medians_abs[0], medians_abs[0]],
linestyle='dashed',
c='dimgrey')
ax.text(start + width / 2.0,
np.min(
light_curve_df[:estimated_time_of_peak_start].irradiance),
'median$_1$ = ' + latex_float(medians[0]) + '% \n' +
'$\sigma_1$ = ' + latex_float(stds[0]) + '%',
fontsize=11,
ha='center',
va='bottom')
# Second window
start = dates.date2num(windows[1].index[0])
end = dates.date2num(windows[1].index[-1])
width = end - start
rect = Rectangle((start, 0), width, 1, color='slateblue', alpha=0.2)
ax.add_patch(rect)
plt.plot([windows[1].index[0], windows[1].index[-1]],
[medians_abs[1], medians_abs[1]],
linestyle='dashed',
c='dimgrey')
ax.text(start + width / 2.0,
np.min(
light_curve_df[:estimated_time_of_peak_start].irradiance),
'median$_2$ = ' + latex_float(medians[1]) + '% \n' +
'$\sigma_2$ = ' + latex_float(stds[1]) + '%',
fontsize=11,
ha='center',
va='bottom')
if not np.isnan(preflare_irradiance):
ax.axes.axhline(y=preflare_irradiance,
linewidth=2,
color='tomato',
linestyle='dashed')
ax.text(
start + width / 2.0,
np.max(
light_curve_df[:estimated_time_of_peak_start].irradiance),
'pre-flare I = ' + latex_float(preflare_irradiance) +
' W m$^{-2}$',
fontsize=11,
ha='center',
va='top',
color='tomato')
else:
ax.text(
start + width / 2.0,
np.max(
light_curve_df[:estimated_time_of_peak_start].irradiance),
'pre-flare I = N/A \n' + 'median condition ok: ' +
str(not failed_median_threshold) + '\n' +
'$\sigma$ condition ok: ' + str(not failed_std_threshold),
fontsize=11,
ha='center',
va='top',
color='tomato')
# Third window
start = dates.date2num(windows[2].index[0])
end = dates.date2num(windows[2].index[-1])
width = end - start
rect = Rectangle((start, 0), width, 1, color='violet', alpha=0.2)
ax.add_patch(rect)
plt.plot([windows[2].index[0], windows[2].index[-1]],
[medians_abs[2], medians_abs[2]],
linestyle='dashed',
c='dimgrey')
ax.text(start + width / 2.0,
np.min(
light_curve_df[:estimated_time_of_peak_start].irradiance),
'median$_3$ = ' + latex_float(medians[2]) + '% \n' +
'$\sigma_3$ = ' + latex_float(stds[2]) + '%',
fontsize=11,
ha='center',
va='bottom')
ax.text(end,
np.max(
light_curve_df[:estimated_time_of_peak_start].irradiance),
'median diff = ' + latex_float(max_median_diff) + '% \n' +
r'thresh $\times \mu_{\sigma n}$ = ' +
latex_float(max_median_diff_threshold * np.mean(stds)) + '%',
fontsize=11,
ha='right',
va='top')
# Increase border so y-axes don't get cut off in savefig, even though they don't in plt.show()
plt.gcf().subplots_adjust(left=0.22)
plt.savefig(plot_path_filename)
if verbose:
logger.info(
"Summary plot for event with start time {0} saved to {1}".
format(estimated_time_of_peak_start, plot_path_filename))
return preflare_irradiance
def generate_jedi_catalog(
flare_index_range, # =range(0, 5052),
threshold_time_prior_flare_minutes=480.0,
dimming_window_relative_to_flare_minutes_left=-1.0,
dimming_window_relative_to_flare_minutes_right=1440.0,
threshold_minimum_dimming_window_minutes=120.0,
output_path='/Users/jmason86/Dropbox/Research/Postdoc_NASA/Analysis/Coronal Dimming Analysis/JEDI Catalog/',
verbose=True):
"""Wrapper code for creating James's Extreme Ultraviolet Variability Experiment (EVE) Dimming Index (JEDI) catalog.
Inputs:
None.
Optional Inputs:
threshold_time_prior_flare_minutes [float]: How long before a particular event does the last one need to have
occurred to be considered independent. If the previous one was too
recent, will use that event's pre-flare irradiance.
The mean dimming time of 100 dimming events in
Reinard and Biesecker (2008, 2009) is the default.
Default is 480 (8 hours).
dimming_window_relative_to_flare_minutes_left [float]: Defines the left side of the time window to search for dimming
relative to the GOES/XRS flare peak. Negative numbers mean
minutes prior to the flare peak. Default is 0.
dimming_window_relative_to_flare_minutes_right [float]: Defines the right side of the time window to search for dimming
relative to the GOES/XRS flare peak. If another flare
occurs before this, that time will define the end of the
window instead. The time that "most" of the 100 dimming events had recovered
by in Reinard and Biesecker (2008, 2009) is the default.
Default is 1440 (24 hours).
threshold_minimum_dimming_window_minutes [float]: The smallest allowed time window in which to search for dimming.
Default is 120 (2 hours).
flare_index_range [range] The range of GOES flare indices to process. Default is range(0, 5052).
output_path [str]: Set to a path for saving the JEDI catalog table and processing
summary plots. Default is '/Users/jmason86/Dropbox/Research/Postdoc_NASA/Analysis/Coronal Dimming Analysis/JEDI Catalog/'.
verbose [bool]: Set to log the processing messages to disk and console. Default is False.
Outputs:
No direct return, but writes a csv to disk with the dimming paramerization results.
Subroutines also optionally save processing plots to disk in output_path.
Optional Outputs:
None
Example:
generate_jedi_catalog(output_path='/Users/jmason86/Dropbox/Research/Postdoc_NASA/Analysis/Coronal Dimming Analysis/JEDI Catalog/',
verbose=True)
"""
# Force flare_index_range to be an array type so it can be indexed in later code
if isinstance(flare_index_range, int):
flare_index_range = np.array([flare_index_range])
# Prepare the logger for verbose
if verbose:
logger = JpmLogger(filename='generate_jedi_catalog',
path=output_path,
console=False)
logger.info("Starting JEDI processing pipeline.")
logger.info("Processing events {0} - {1}".format(
flare_index_range[0], flare_index_range[-1]))
else:
logger = None
# Get EVE level 2 extracted emission lines data
# TODO: Replace this shortcut method with the method I'm building into sunpy
from scipy.io.idl import readsav
eve_readsav = readsav(
'/Users/jmason86/Dropbox/Research/Data/EVE/eve_lines_2010121-2014146 MEGS-A Mission Bare Bones.sav'
)
if verbose:
logger.info('Loaded EVE data')
# Create metadata dictionary
# TODO: Replace this shortcut method with the method I'm building into sunpy
from sunpy.util.metadata import MetaDict
metadata = MetaDict()
metadata['ion'] = eve_readsav['name']
metadata['temperature_ion_peak_formation'] = np.power(
10.0, eve_readsav['logt']) * u.Kelvin
metadata['extracted_wavelength_center'] = eve_readsav['wavelength'] * u.nm
metadata['extracted_wavelength_min'] = metadata[
'extracted_wavelength_center']
metadata['extracted_wavelength_max'] = metadata[
'extracted_wavelength_center']
metadata['emission_line_blends'] = ['none', 'yay', 'poop', 'Fe vi'] # etc
metadata[
'exposure_time'] = 60.0 * u.second # These example EVE data are already binned down to 1 minute
metadata['precision'] = ['Not implemented in prototype']
metadata['accuracy'] = ['Not implemented in prototype']
metadata['flags'] = ['Not implemented in prototype']
metadata['flags_description'] = '1 = MEGS-A data is missing, ' \
'2 = MEGS-B data is missing, ' \
'4 = ESP data is missing, ' \
'8 = MEGS-P data is missing, ' \
'16 = Possible clock adjust in MEGS-A, ' \
'32 = Possible clock adjust in MEGS-B, ' \
'64 = Possible clock adjust in ESP, ' \
'128 = Possible clock adjust in MEGS-P'
metadata['flags_spacecraft'] = ['Not implemented in prototype']
metadata['flags_spacecraft_description'] = '0 = No obstruction, ' \
'1 = Warm up from Earth eclipse, ' \
'2 = Obstruction atmosphere penumbra, ' \
'3 = Obstruction atmosphere umbra, ' \
'4 = Obstruction penumbra of Mercury, ' \
'5 = Obstruction penumbra of Mercury, ' \
'6 = Obstruction penumbra of Venus, ' \
'7 = Obstruction umbra of Venus, ' \
'8 = Obstruction penumbra of Moon, ' \
'9 = Obstruction umbra of Moon, ' \
'10 = Obstruction penumbra of solid Earth, ' \
'11 = Obstruction umbra of solid Earth, ' \
'16 = Observatory is off-pointed by more than 1 arcmin'
metadata['data_version'] = ['Not implemented in prototype']
metadata['data_reprocessed_revision'] = ['Not implemented in prototype']
metadata['filename'] = ['Not implemented in prototype']
# Load up the actual irradiance data into a pandas DataFrame
# TODO: Replace this shortcut method with the method I'm building into sunpy
irradiance = eve_readsav['irradiance'].byteswap().newbyteorder(
) # pandas doesn't like big endian
irradiance[irradiance == -1] = np.nan
wavelengths = eve_readsav['wavelength']
wavelengths_str = []
[
wavelengths_str.append('{0:1.1f}'.format(wavelength))
for wavelength in wavelengths
]
eve_lines = pd.DataFrame(irradiance, columns=wavelengths_str)
eve_lines.index = pd.to_datetime(eve_readsav.iso.astype(str))
eve_lines.sort_index(inplace=True)
eve_lines = eve_lines.drop_duplicates()
# Get GOES flare events above C1 within date range corresponding to EVE data
# flares = get_goes_flare_events(eve_lines.index[0], eve_lines.index[-1], verbose=verbose) # TODO: The method in sunpy needs fixing, issue 2434
# Load GOES events from IDL saveset instead of directly through sunpy
goes_flare_events = readsav(
'/Users/jmason86/Dropbox/Research/Data/GOES/events/GoesEventsC1MinMegsAEra.sav'
)
goes_flare_events['class'] = goes_flare_events['class'].astype(str)
goes_flare_events['event_peak_time_human'] = goes_flare_events[
'event_peak_time_human'].astype(str)
goes_flare_events['event_start_time_human'] = goes_flare_events[
'event_start_time_human'].astype(str)
goes_flare_events['peak_time'] = Time(
goes_flare_events['event_peak_time_jd'], format='jd', scale='utc')
goes_flare_events['start_time'] = Time(
goes_flare_events['event_start_time_jd'], format='jd', scale='utc')
if verbose:
logger.info('Loaded GOES flare events.')
# Define the columns of the JEDI catalog
jedi_row = pd.DataFrame([
OrderedDict([('Event #', np.nan), ('GOES Flare Start Time', np.nan),
('GOES Flare Peak Time', np.nan),
('GOES Flare Class', np.nan),
('Pre-Flare Start Time', np.nan),
('Pre-Flare End Time', np.nan),
('Flare Interrupt', np.nan)])
])
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns +
' Pre-Flare Irradiance [W/m2]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Min [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Max [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Mean [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Slope Uncertainty [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Depth Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Depth [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Depth Uncertainty [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Duration Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Duration End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Duration [s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Fitting Gamma'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=eve_lines.columns + ' Fitting Score'))
ion_tuples = list(itertools.permutations(eve_lines.columns.values, 2))
ion_permutations = pd.Index(
[' by '.join(ion_tuples[i]) for i in range(len(ion_tuples))])
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Min [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Max [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Mean [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Slope Uncertainty [%/s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Depth Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Depth [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Depth Uncertainty [%]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Duration Start Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Duration End Time'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Duration [s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Correction Time Shift [s]'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Correction Scale Factor'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Fitting Gamma'))
jedi_row = jedi_row.join(
pd.DataFrame(columns=ion_permutations + ' Fitting Score'))
csv_filename = output_path + 'jedi_{0}.csv'.format(Time.now().iso)
jedi_row.to_csv(csv_filename, header=True, index=False, mode='w')
if verbose:
logger.info('Created JEDI row definition.')
# Start a progress bar
#widgets = [progressbar.Percentage(), progressbar.Bar(), progressbar.Timer(), ' ', progressbar.AdaptiveETA()]
#progress_bar = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('Flare event loop: ')] + widgets,
# min_value=flare_index_range[0], max_value=flare_index_range[-1]).start()
# Prepare a hold-over pre-flare irradiance value,
# which will normally have one element for each of the 39 emission lines
preflare_irradiance = np.nan
# Start loop through all flares
for flare_index in flare_index_range:
loop_time = time.time()
# Skip event 0 to avoid problems with referring to earlier indices
if flare_index == 0:
continue
print('Running on event {0}'.format(flare_index))
# Reset jedi_row
jedi_row[:] = np.nan
# Reset the flare interrupt flag
flare_interrupt = False
# Fill the GOES flare information into the JEDI row
jedi_row['Event #'] = flare_index
jedi_row['GOES Flare Start Time'] = goes_flare_events['start_time'][
flare_index].iso
jedi_row['GOES Flare Peak Time'] = goes_flare_events['peak_time'][
flare_index].iso
jedi_row['GOES Flare Class'] = goes_flare_events['class'][flare_index]
if verbose:
logger.info(
"Event {0} GOES flare details stored to JEDI row.".format(
flare_index))
# If haven't already done all pre-parameterization processing
processed_jedi_non_params_filename = output_path + 'Processed Pre-Parameterization Data/Event {0} Pre-Parameterization.h5'.format(
flare_index)
processed_lines_filename = output_path + 'Processed Lines Data/Event {0} Lines.h5'.format(
flare_index)
if not os.path.isfile(processed_lines_filename) or not os.path.isfile(
processed_jedi_non_params_filename):
# Determine pre-flare irradiance
minutes_since_last_flare = (
goes_flare_events['peak_time'][flare_index] -
goes_flare_events['peak_time'][flare_index - 1]).sec / 60.0
if minutes_since_last_flare > threshold_time_prior_flare_minutes:
# Clip EVE data from threshold_time_prior_flare_minutes prior to flare up to peak flare time
preflare_window_start = (
goes_flare_events['peak_time'][flare_index] -
(threshold_time_prior_flare_minutes * u.minute)).iso
preflare_window_end = (
goes_flare_events['peak_time'][flare_index]).iso
eve_lines_preflare_time = eve_lines[
preflare_window_start:preflare_window_end]
# Loop through the emission lines and get pre-flare irradiance for each
preflare_irradiance = []
for column in eve_lines_preflare_time:
eve_line_preflare_time = pd.DataFrame(
eve_lines_preflare_time[column])
eve_line_preflare_time.columns = ['irradiance']
preflare_irradiance.append(
determine_preflare_irradiance(
eve_line_preflare_time,
pd.Timestamp(goes_flare_events['start_time']
[flare_index].iso),
plot_path_filename=output_path +
'Preflare Determination/Event {0} {1}.png'.format(
flare_index, column),
verbose=verbose,
logger=logger))
plt.close('all')
else:
logger.info(
"This flare at {0} will use the pre-flare irradiance from flare at {1}."
.format(
goes_flare_events['peak_time'][flare_index].iso,
goes_flare_events['peak_time'][flare_index - 1].iso))
# Make sure there is a preflare_window_start defined. If not, user probably needs to start from an earlier event.
if 'preflare_window_start' not in locals():
print(
'ERROR: You need to start from an earlier event. Detected flare interrupt so need earlier pre-flare value.'
)
logger.error(
'ERROR: You need to start from an earlier event. Detected flare interrupt so need earlier pre-flare value.'
)
print('Moving to next event instead.')
continue
jedi_row["Pre-Flare Start Time"] = preflare_window_start
jedi_row["Pre-Flare End Time"] = preflare_window_end
preflare_irradiance_cols = [
col for col in jedi_row.columns
if 'Pre-Flare Irradiance' in col
]
jedi_row[preflare_irradiance_cols] = preflare_irradiance
if verbose:
logger.info(
"Event {0} pre-flare determination complete.".format(
flare_index))
# Clip EVE data to dimming window
bracket_time_left = (
goes_flare_events['peak_time'][flare_index] +
(dimming_window_relative_to_flare_minutes_left * u.minute))
next_flare_time = Time(
(goes_flare_events['peak_time'][flare_index + 1]).iso)
user_choice_time = (
goes_flare_events['peak_time'][flare_index] +
(dimming_window_relative_to_flare_minutes_right * u.minute))
bracket_time_right = min(next_flare_time, user_choice_time)
# If flare is shortening the window, set the flare_interrupt flag
if bracket_time_right == next_flare_time:
flare_interrupt = True
if verbose:
logger.info(
'Flare interrupt for event at {0} by flare at {1}'.
format(goes_flare_events['peak_time'][flare_index].iso,
next_flare_time))
# Write flare_interrupt to JEDI row
jedi_row['Flare Interrupt'] = flare_interrupt
# Skip event if the dimming window is too short
if ((bracket_time_right - bracket_time_left).sec /
60.0) < threshold_minimum_dimming_window_minutes:
# Leave all dimming parameters as NaN and write this null result to the CSV on disk
jedi_row.to_csv(csv_filename,
header=False,
index=False,
mode='a')
# Log message
if verbose:
logger.info(
'The dimming window duration of {0} minutes is shorter than the minimum threshold of {1} minutes. Skipping this event ({2})'
.format(((bracket_time_right - bracket_time_left).sec /
60.0),
threshold_minimum_dimming_window_minutes,
goes_flare_events['peak_time'][flare_index]))
# Skip the rest of the processing in the flare_index loop
continue
else:
eve_lines_event = eve_lines[bracket_time_left.
iso:bracket_time_right.iso]
if verbose:
logger.info(
"Event {0} EVE data clipped to dimming window.".format(
flare_index))
# Convert irradiance units to percent
# (in place, don't care about absolute units from this point forward)
eve_lines_event = (eve_lines_event - preflare_irradiance
) / preflare_irradiance * 100.0
if verbose:
logger.info(
"Event {0} irradiance converted from absolute to percent units."
.format(flare_index))
# Do flare removal in the light curves and add the results to the DataFrame
#progress_bar_correction = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('Peak match subtract: ')] + widgets,
# max_value=len(ion_tuples)).start()
time_correction = time.time()
for i in range(len(ion_tuples)):
light_curve_to_subtract_from_df = pd.DataFrame(
eve_lines_event[ion_tuples[i][0]])
light_curve_to_subtract_from_df.columns = ['irradiance']
light_curve_to_subtract_with_df = pd.DataFrame(
eve_lines_event[ion_tuples[i][1]])
light_curve_to_subtract_with_df.columns = ['irradiance']
if (light_curve_to_subtract_from_df.isnull().all().all()) or (
light_curve_to_subtract_with_df.isnull().all().all()):
if verbose:
logger.info(
'Event {0} {1} correction skipped because all irradiances are NaN.'
.format(flare_index, ion_permutations[i]))
else:
light_curve_corrected, seconds_shift, scale_factor = light_curve_peak_match_subtract(
light_curve_to_subtract_from_df,
light_curve_to_subtract_with_df,
pd.Timestamp(
(goes_flare_events['peak_time'][flare_index]).iso),
plot_path_filename=output_path +
'Peak Subtractions/Event {0} {1}.png'.format(
flare_index, ion_permutations[i]),
verbose=verbose,
logger=logger)
eve_lines_event[
ion_permutations[i]] = light_curve_corrected
jedi_row[ion_permutations[i] +
' Correction Time Shift [s]'] = seconds_shift
jedi_row[ion_permutations[i] +
' Correction Scale Factor'] = scale_factor
plt.close('all')
if verbose:
logger.info(
'Event {0} flare removal correction complete'.
format(flare_index))
#progress_bar_correction.update(i)
#progress_bar_correction.finish()
print('Time to do peak match subtract [s]: {0}'.format(
time.time() - time_correction))
# TODO: Update calculate_eve_fe_line_precision to compute for all emission lines, not just selected
uncertainty = np.ones(len(eve_lines_event)) * 0.002545
# TODO: Propagate uncertainty through light_curve_peak_match_subtract and store in eve_lines_event
# Fit the light curves to reduce influence of noise on the parameterizations to come later
#progress_bar_fitting = progressbar.ProgressBar(widgets=[progressbar.FormatLabel('Light curve fitting: ')] + widgets,
# max_value=len(eve_lines_event.columns)).start()
time_fitting = time.time()
for i, column in enumerate(eve_lines_event):
if eve_lines_event[column].isnull().all().all():
if verbose:
logger.info(
'Event {0} {1} fitting skipped because all irradiances are NaN.'
.format(flare_index, column))
else:
eve_line_event = pd.DataFrame(eve_lines_event[column])
eve_line_event.columns = ['irradiance']
eve_line_event['uncertainty'] = uncertainty
fitting_path = output_path + 'Fitting/'
if not os.path.exists(fitting_path):
os.makedirs(fitting_path)
plt.close('all')
light_curve_fit_df, best_fit_gamma, best_fit_score = light_curve_fit(
eve_line_event,
gamma=np.array([5e-8]),
plots_save_path='{0}Event {1} {2} '.format(
fitting_path, flare_index, column),
verbose=verbose,
logger=logger)
eve_lines_event[column] = light_curve_fit_df
jedi_row[column + ' Fitting Gamma'] = best_fit_gamma
jedi_row[column + ' Fitting Score'] = best_fit_score
if verbose:
logger.info(
'Event {0} {1} light curves fitted.'.format(
flare_index, column))
#progress_bar_fitting.update(i)
#progress_bar_fitting.finish()
print('Time to do fitting [s]: {0}'.format(time.time() -
time_fitting))
# Save the dimming event data to disk for quicker restore
jedi_row.to_hdf(processed_jedi_non_params_filename, 'jedi_row')
eve_lines_event.to_hdf(processed_lines_filename, 'eve_lines_event')
else:
jedi_row = pd.read_hdf(processed_jedi_non_params_filename,
'jedi_row')
eve_lines_event = pd.read_hdf(processed_lines_filename,
'eve_lines_event')
if verbose:
logger.info(
'Loading files {0} and {1} rather than processing again.'.
format(processed_jedi_non_params_filename,
processed_lines_filename))
# Parameterize the light curves for dimming
for column in eve_lines_event:
# Null out all parameters
depth_percent, depth_time = np.nan, np.nan
slope_start_time, slope_end_time = np.nan, np.nan
slope_min, slope_max, slope_mean = np.nan, np.nan, np.nan
duration_seconds, duration_start_time, duration_end_time = np.nan, np.nan, np.nan
# Determine whether to do the parameterizations or not
if eve_lines_event[column].isnull().all().all():
if verbose:
logger.info(
'Event {0} {1} parameterization skipped because all irradiances are NaN.'
.format(flare_index, column))
else:
eve_line_event = pd.DataFrame(eve_lines_event[column])
eve_line_event.columns = ['irradiance']
# Determine dimming depth (if any)
depth_path = output_path + 'Depth/'
if not os.path.exists(depth_path):
os.makedirs(depth_path)
plt.close('all')
depth_percent, depth_time = determine_dimming_depth(
eve_line_event,
plot_path_filename='{0}Event {1} {2} Depth.png'.format(
depth_path, flare_index, column),
verbose=verbose,
logger=logger)
jedi_row[column + ' Depth [%]'] = depth_percent
# jedi_row[column + ' Depth Uncertainty [%]'] = depth_uncertainty # TODO: make determine_dimming_depth return the propagated uncertainty
jedi_row[column + ' Depth Time'] = depth_time
# Determine dimming slope (if any)
slope_path = output_path + 'Slope/'
if not os.path.exists(slope_path):
os.makedirs(slope_path)
slope_start_time = pd.Timestamp(
(goes_flare_events['peak_time'][flare_index]).iso)
slope_end_time = depth_time
if (pd.isnull(slope_start_time)) or (
pd.isnull(slope_end_time)):
if verbose:
logger.warning(
'Cannot compute slope or duration because slope bounding times NaN.'
)
else:
plt.close('all')
slope_min, slope_max, slope_mean = determine_dimming_slope(
eve_line_event,
earliest_allowed_time=slope_start_time,
latest_allowed_time=slope_end_time,
plot_path_filename='{0}Event {1} {2} Slope.png'.format(
slope_path, flare_index, column),
verbose=verbose,
logger=logger)
jedi_row[column + ' Slope Min [%/s]'] = slope_min
jedi_row[column + ' Slope Max [%/s]'] = slope_max
jedi_row[column + ' Slope Mean [%/s]'] = slope_mean
# jedi_row[column + ' Slope Uncertainty [%]'] = slope_uncertainty # TODO: make determine_dimming_depth return the propagated uncertainty
jedi_row[column + ' Slope Start Time'] = slope_start_time
jedi_row[column + ' Slope End Time'] = slope_end_time
# Determine dimming duration (if any)
duration_path = output_path + 'Duration/'
if not os.path.exists(duration_path):
os.makedirs(duration_path)
plt.close('all')
duration_seconds, duration_start_time, duration_end_time = determine_dimming_duration(
eve_line_event,
earliest_allowed_time=slope_start_time,
plot_path_filename='{0}Event {1} {2} Duration.png'.
format(duration_path, flare_index, column),
verbose=verbose,
logger=logger)
jedi_row[column + ' Duration [s]'] = duration_seconds
jedi_row[column +
' Duration Start Time'] = duration_start_time
jedi_row[column + ' Duration End Time'] = duration_end_time
if verbose:
logger.info(
"Event {0} {1} parameterizations complete.".format(
flare_index, column))
# Produce a summary plot for each light curve
plt.style.use('jpm-transparent-light')
ax = eve_line_event['irradiance'].plot(color='black')
plt.axhline(linestyle='dashed', color='grey')
start_date = eve_line_event.index.values[0]
start_date_string = pd.to_datetime(str(start_date))
plt.xlabel(start_date_string.strftime('%Y-%m-%d %H:%M:%S'))
plt.ylabel('Irradiance [%]')
fmtr = dates.DateFormatter("%H:%M:%S")
ax.xaxis.set_major_formatter(fmtr)
ax.xaxis.set_major_locator(dates.HourLocator())
plt.title('Event {0} {1} nm Parameters'.format(
flare_index, column))
if not np.isnan(depth_percent):
plt.annotate('',
xy=(depth_time, -depth_percent),
xycoords='data',
xytext=(depth_time, 0),
textcoords='data',
arrowprops=dict(facecolor='limegreen',
edgecolor='limegreen',
linewidth=2))
mid_depth = -depth_percent / 2.0
plt.annotate('{0:.2f} %'.format(depth_percent),
xy=(depth_time, mid_depth),
xycoords='data',
ha='right',
va='center',
rotation=90,
size=18,
color='limegreen')
if not np.isnan(slope_mean):
p = plt.plot(
eve_line_event[slope_start_time:slope_end_time]
['irradiance'],
c='tomato')
inverse_str = '$^{-1}$'
plt.annotate('slope_min={0} % s{1}'.format(
latex_float(slope_min), inverse_str),
xy=(0.98, 0.12),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
plt.annotate('slope_max={0} % s{1}'.format(
latex_float(slope_max), inverse_str),
xy=(0.98, 0.08),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
plt.annotate('slope_mean={0} % s{1}'.format(
latex_float(slope_mean), inverse_str),
xy=(0.98, 0.04),
xycoords='axes fraction',
ha='right',
size=12,
color=p[0].get_color())
if not np.isnan(duration_seconds):
plt.annotate('',
xy=(duration_start_time, 0),
xycoords='data',
xytext=(duration_end_time, 0),
textcoords='data',
arrowprops=dict(facecolor='dodgerblue',
edgecolor='dodgerblue',
linewidth=5,
arrowstyle='<->'))
mid_time = duration_start_time + (duration_end_time -
duration_start_time) / 2
plt.annotate(str(duration_seconds) + ' s',
xy=(mid_time, 0),
xycoords='data',
ha='center',
va='bottom',
size=18,
color='dodgerblue')
summary_path = output_path + 'Summary Plots/'
if not os.path.exists(summary_path):
os.makedirs(summary_path)
summary_filename = '{0}Event {1} {2} Parameter Summary.png'.format(
summary_path, flare_index, column)
plt.savefig(summary_filename)
plt.close('all')
if verbose:
logger.info("Summary plot saved to %s" % summary_filename)
# Write to the JEDI catalog on disk
jedi_row.to_csv(csv_filename, header=False, index=False, mode='a')
if verbose:
logger.info('Event {0} JEDI row written to {1}.'.format(
flare_index, csv_filename))
# Update progress bar
#progress_bar.update(flare_index)
print('Total time for loop [s]: {0}'.format(time.time() - loop_time))