def __init__(self, data, frequency="W",
date_method="expanding", rollyears=20,
dict_group=dict(), boring_offdiag=0.99, cleaning=True, **kwargs):
cleaning=str2Bool(cleaning)
group_dict=group_dict_from_natural(dict_group)
data=df_from_list(data)
column_names=list(data.columns)
data=data.resample(frequency, how="last")
fit_dates = generate_fitting_dates(data, date_method=date_method, rollyears=rollyears)
size=len(column_names)
corr_with_no_data=boring_corr_matrix(size, offdiag=boring_offdiag)
corr_list=[]
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Correlation estimate")
for fit_period in fit_dates:
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Estimating from %s to %s" % (fit_period.period_start, fit_period.period_end))
if fit_period.no_data:
corr_with_nan=boring_corr_matrix(size, offdiag=np.nan, diag=np.nan)
corrmat=corr_with_nan
else:
data_for_estimate=data[fit_period.fit_start:fit_period.fit_end]
corrmat=correlation_single_period(data_for_estimate, **kwargs)
if cleaning:
current_period_data=data[fit_period.fit_start:fit_period.fit_end]
must_haves=must_have_item(current_period_data)
corrmat=clean_correlation(corrmat, corr_with_no_data, must_haves)
corr_list.append(corrmat)
setattr(self, "corr_list", corr_list)
setattr(self, "columns", column_names)
setattr(self, "fit_dates", fit_dates)
def __init__(self, data, log=logtoscreen("optimiser"), frequency="W", date_method="expanding",
rollyears=20,
dict_group=dict(), boring_offdiag=0.99, cleaning=True, **kwargs):
cleaning=str2Bool(cleaning)
## grouping dictionary, convert to faster, algo friendly, form
group_dict=group_dict_from_natural(dict_group)
data=df_from_list(data)
column_names=list(data.columns)
data=data.resample(frequency, how="last")
### Generate time periods
fit_dates = generate_fitting_dates(data, date_method=date_method, rollyears=rollyears)
size=len(column_names)
corr_with_no_data=boring_corr_matrix(size, offdiag=boring_offdiag)
## create a list of correlation matrices
corr_list=[]
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Correlation estimate")
## Now for each time period, estimate correlation
for fit_period in fit_dates:
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Estimating from %s to %s" % (fit_period.period_start, fit_period.period_end))
if fit_period.no_data:
## no data to fit with
corr_with_nan=boring_corr_matrix(size, offdiag=np.nan, diag=np.nan)
corrmat=corr_with_nan
else:
data_for_estimate=data[fit_period.fit_start:fit_period.fit_end]
corrmat=correlation_single_period(data_for_estimate,
**kwargs)
if cleaning:
current_period_data=data[fit_period.fit_start:fit_period.fit_end]
must_haves=must_have_item(current_period_data)
# means we can use earlier correlations with sensible values
corrmat=clean_correlation(corrmat, corr_with_no_data, must_haves)
corr_list.append(corrmat)
setattr(self, "corr_list", corr_list)
setattr(self, "columns", column_names)
setattr(self, "fit_dates", fit_dates)
def __init__(self,
data,
log=logtoscreen("optimiser"),
frequency="W",
date_method="expanding",
rollyears=20,
dict_group=dict(),
boring_offdiag=0.99,
cleaning=True,
**kwargs):
"""
We generate a correlation from either a pd.DataFrame, or a list of them if we're pooling
Its important that forward filling, or index / ffill / diff has been done before we begin
:param data: Data to get correlations from
:type data: pd.DataFrame or list if pooling
:param frequency: Downsampling frequency. Must be "D", "W" or bigger
:type frequency: str
:param date_method: Method to pass to generate_fitting_dates
:type date_method: str
:param roll_years: If date_method is "rolling", number of years in window
:type roll_years: int
:param dict_group: dictionary of groupings; used to replace missing values
:type dict_group: dict
:param boring_offdiag: Value used in creating 'boring' matrix, for when no data
:type boring_offdiag: float
:param **kwargs: passed to correlation_single_period
:returns: CorrelationList
"""
cleaning = str2Bool(cleaning)
# grouping dictionary, convert to faster, algo friendly, form
group_dict = group_dict_from_natural(dict_group)
data = df_from_list(data)
column_names = list(data.columns)
data = data.resample(frequency).last()
# Generate time periods
fit_dates = generate_fitting_dates(data,
date_method=date_method,
rollyears=rollyears)
size = len(column_names)
corr_with_no_data = boring_corr_matrix(size, offdiag=boring_offdiag)
# create a list of correlation matrices
corr_list = []
progress = progressBar(len(fit_dates), "Estimating correlations")
# Now for each time period, estimate correlation
for fit_period in fit_dates:
progress.iterate()
if fit_period.no_data:
# no data to fit with
corr_with_nan = boring_corr_matrix(size,
offdiag=np.nan,
diag=np.nan)
corrmat = corr_with_nan
else:
data_for_estimate = data[fit_period.fit_start:fit_period.
fit_end]
corrmat = correlation_single_period(data_for_estimate,
**kwargs)
if cleaning:
current_period_data = data[fit_period.fit_start:fit_period.
fit_end]
must_haves = must_have_item(current_period_data)
# means we can use earlier correlations with sensible values
corrmat = clean_correlation(corrmat, corr_with_no_data,
must_haves)
corr_list.append(corrmat)
setattr(self, "corr_list", corr_list)
setattr(self, "columns", column_names)
setattr(self, "fit_dates", fit_dates)
def __init__(self, data, log=logtoscreen("optimiser"), frequency="W", date_method="expanding", rollyears=20,
dict_group=dict(), boring_offdiag=0.99, cleaning=True, **kwargs):
"""
We generate a correlation from eithier a pd.DataFrame, or a list of them if we're pooling
Its important that forward filling, or index / ffill / diff has been done before we begin
:param data: Data to get correlations from
:type data: pd.DataFrame or list if pooling
:param frequency: Downsampling frequency. Must be "D", "W" or bigger
:type frequency: str
:param date_method: Method to pass to generate_fitting_dates
:type date_method: str
:param roll_years: If date_method is "rolling", number of years in window
:type roll_years: int
:param dict_group: dictionary of groupings; used to replace missing values
:type dict_group: dict
:param boring_offdiag: Value used in creating 'boring' matrix, for when no data
:type boring_offdiag: float
:param **kwargs: passed to correlation_single_period
:returns: CorrelationList
"""
cleaning=str2Bool(cleaning)
## grouping dictionary, convert to faster, algo friendly, form
group_dict=group_dict_from_natural(dict_group)
data=df_from_list(data)
column_names=list(data.columns)
data=data.resample(frequency, how="last")
### Generate time periods
fit_dates = generate_fitting_dates(data, date_method=date_method, rollyears=rollyears)
size=len(column_names)
corr_with_no_data=boring_corr_matrix(size, offdiag=boring_offdiag)
## create a list of correlation matrices
corr_list=[]
log.terse("Correlation estimate")
## Now for each time period, estimate correlation
for fit_period in fit_dates:
log.msg("Fitting from %s to %s" % (fit_period.period_start, fit_period.period_end))
if fit_period.no_data:
## no data to fit with
corr_with_nan=boring_corr_matrix(size, offdiag=np.nan, diag=np.nan)
corrmat=corr_with_nan
else:
data_for_estimate=data[fit_period.fit_start:fit_period.fit_end]
corrmat=correlation_single_period(data_for_estimate,
**kwargs)
if cleaning:
# means we can use earlier correlations with sensible values
corrmat=clean_correlation(corrmat, corr_with_no_data, boring_offdiag)
corr_list.append(corrmat)
setattr(self, "corr_list", corr_list)
setattr(self, "columns", column_names)
setattr(self, "fit_dates", fit_dates)