def prepare_data_sim(df, downsample=False):
# Clean and convert data
df_sim_clean = df.dropna(subset=['Sim_Cosine', 'Sim_Jaccard'])
sec_filing_date = pd.to_datetime(df_sim_clean['sec_filing_date'],
format='%Y%m%d')
df_sim_clean['sec_filing_date'] = sec_filing_date
df_sim_clean = df_sim_clean[
df_sim_clean['sec_filing_date'] < datetime.datetime(2019, 1, 1)]
df_sim_clean = df_sim_clean[
df_sim_clean['sec_filing_date'] > datetime.datetime(2006, 12, 31)]
# Create train & Validation set
df_sim_clean = df_sim_clean[[
'rating_downgrade', 'sec_year', 'Sim_Cosine', 'Sim_Jaccard', 'default',
'sec_filing_date'
]]
df_sim_clean = df_sim_clean.dropna()
df_sim_clean = calculate_previous_quantile(df_sim_clean, 5,
['Sim_Jaccard', 'Sim_Cosine'])
df_sim_clean = df_sim_clean.dropna()
# Create holdoutset
df_holdout = df_sim_clean[
df_sim_clean['sec_filing_date'] > datetime.datetime(2017, 12, 31)]
if downsample:
df_holdout = down_sample_majority_class(df_holdout, 'rating_downgrade')
# Create train & Validation set
df_train = df_sim_clean[
df_sim_clean['sec_filing_date'] <= datetime.datetime(2017, 12, 31)]
if downsample:
df_train = down_sample_majority_class(df_train, 'rating_downgrade')
return df_train, df_holdout
def prepare_data_avg_embedding(df, downsample=False):
"""
Title: Learning Word Embeddings from 10-K Filings for Financial NLP Tasks
Author: Saurabh Sehrawat
Date: 2019
Code version: 1.0
Availability: https://github.com/ssehrawat/10K-word-embeddings
"""
embed = torch.load(main_dir +
'data/10K-word-embeddings/10k_word_embeddings.tar')
vocab_to_int = torch.load(main_dir +
'data/10K-word-embeddings/vocab_to_int.tar')
df = df.dropna(subset=['diff_text'])
avg_embd = np.zeros([1, 300], dtype=float)
embd_matrix = np.zeros([1, 300], dtype=float)
for index, row in tqdm(df.iterrows()):
doc = row['diff_text']
doc = doc.lower()
doc = doc.split()
embd_array = np.zeros([1, 300], dtype=float)
for word in doc:
if word in vocab_to_int.keys():
word_emd = embed[vocab_to_int[word]]
embd_array = np.append(embd_array,
word_emd.reshape(1, 300),
axis=0)
avg_embd = np.average(embd_array[1:], axis=0).reshape(1, 300)
embd_matrix = np.concatenate((embd_matrix, avg_embd), axis=0)
data = pd.DataFrame(embd_matrix[1:])
sec_filing_date = pd.to_datetime(df['sec_filing_date'], format='%Y%m%d')
data['sec_filing_date'] = sec_filing_date
data['rating_downgrade'] = df['rating_downgrade']
# Create train & Validation set
data = data.dropna()
data = data[data['sec_filing_date'] < datetime.datetime(2019, 1, 1)]
# Create holdoutset
df_holdout = data[
data['sec_filing_date'] > datetime.datetime(2017, 12, 31)]
if downsample:
df_holdout = down_sample_majority_class(df_holdout, 'rating_downgrade')
# Create train & Validation set
df_train = data[data['sec_filing_date'] <= datetime.datetime(2017, 12, 31)]
if downsample:
df_train = down_sample_majority_class(df_train, 'rating_downgrade')
return df_train, df_holdout, data
def prepare_data_tfidf(df, downsample=False):
df = df.dropna(subset=['diff_text'])
stemmer = WordNetLemmatizer()
docs = []
for index, row in df.iterrows():
doc = row['diff_text']
doc = doc.lower()
doc = doc.split()
doc = [stemmer.lemmatize(word) for word in doc]
doc = ' '.join(doc)
docs.append(doc)
tfidf = TfidfVectorizer(max_features=1500,
min_df=5,
max_df=0.7,
stop_words=stopwords.words('english'))
X = tfidf.fit_transform(docs).toarray()
data = pd.DataFrame(X)
sec_filing_date = pd.to_datetime(df['sec_filing_date'], format='%Y%m%d')
data['sec_filing_date'] = sec_filing_date
data['rating_downgrade'] = df['rating_downgrade']
# Create train & Validation set
data = data.dropna()
data = data[data['sec_filing_date'] < datetime.datetime(2019, 1, 1)]
# Create holdoutset
df_holdout = data[
data['sec_filing_date'] > datetime.datetime(2017, 12, 31)]
if downsample:
df_holdout = down_sample_majority_class(df_holdout, 'rating_downgrade')
# Create train & Validation set
df_train = data[data['sec_filing_date'] <= datetime.datetime(2017, 12, 31)]
if downsample:
df_train = down_sample_majority_class(df_train, 'rating_downgrade')
return df_train, df_holdout
def prepare_data_LDA(df, text_feature, downsample=False):
# Clean and convert data
data = df.dropna(subset=[text_feature])
sec_filing_date = pd.to_datetime(data['sec_filing_date'], format='%Y%m%d')
data['sec_filing_date'] = sec_filing_date
data['rating_downgrade'] = data['rating_downgrade']
# Create train & Validation set
data = data[data['sec_filing_date'] < datetime.datetime(2019, 1, 1)]
# Create holdoutset
df_holdout = data[
data['sec_filing_date'] > datetime.datetime(2017, 12, 31)]
if downsample:
df_holdout = down_sample_majority_class(df_holdout, 'rating_downgrade')
# Create train & Validation set
df_train = data[data['sec_filing_date'] <= datetime.datetime(2017, 12, 31)]
if downsample:
df_train = down_sample_majority_class(df_train, 'rating_downgrade')
return df_train, df_holdout
def model_training_out_of_time(df,
holdout,
target,
features,
algo,
standard=True,
show_holdout=False,
downsample=False):
X = df[features]
y = df[target].astype('bool')
X_holdout = holdout[features].drop('sec_filing_date', axis=1)
y_holdout = holdout[target].astype('bool')
if standard:
clf = make_pipeline(StandardScaler(), algo)
else:
clf = algo
alog_name = str(clf.steps[1][1]).split('(')[0]
alog_name = " ".join(re.findall('[A-Z][^A-Z]*', alog_name))
if alog_name == 'S V C':
alog_name = 'SVM Classifier'
print('### ' + alog_name + ' ###')
scores = {'acc': [], 'f1': []}
cf_matrix_val = np.zeros((2, 2), dtype=np.int)
tbcv = TimeBasedCV(train_period=3, test_period=1, freq='years')
tbcv_folds = tbcv.split(df,
validation_split_date=datetime.date(2008, 12, 31),
date_column='sec_filing_date')
for train_index, test_index in tbcv_folds:
if downsample:
df_kfold = down_sample_majority_class(df, 'rating_downgrade')
df_kfold_index = df_kfold.index.tolist()
train_index = [
idx for idx in list(train_index) if idx in df_kfold_index
]
data_train = X.loc[train_index].drop('sec_filing_date', axis=1)
target_train = y.loc[train_index]
data_test = X.loc[test_index].drop('sec_filing_date', axis=1)
target_test = y.loc[test_index]
clf.fit(data_train, target_train.values.ravel())
preds = clf.predict(data_test)
# accuracy for the current fold only
score = clf.score(data_test, target_test)
f1 = f1_score(target_test, preds)
cf_matrix_val += confusion_matrix(target_test, preds)
scores['acc'].append(score)
scores['f1'].append(f1)
print("Cross Validation Score: " +
str(sum(scores['acc']) / len(scores['acc'])))
if show_holdout:
# Test model trained on last three years on holdout data
frames = [test_index for train_index, test_index in tbcv_folds[-3:]]
frames = [item for sublist in frames for item in sublist]
data_train = X.loc[frames].drop('sec_filing_date', axis=1)
target_train = y.loc[frames]
clf.fit(data_train, target_train.values.ravel())
holdout_preds = clf.predict(X_holdout)
cf_matrix = confusion_matrix(y_holdout, holdout_preds)
print("Holdout Score: " + str(clf.score(X_holdout, y_holdout)))
print('\n')
# Visualize confusion matrix for holdout data
labels = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
categories = ['No Downgrade', 'Downgrade']
make_confusion_matrix(cf_matrix,
group_names=labels,
categories=categories,
cbar=False,
title='Confusion Matrix: ' + alog_name,
figsize=(10, 10))
else:
#Visualize confusion matrix for cross-val data
labels = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
categories = ['No Downgrade', 'Downgrade']
make_confusion_matrix(cf_matrix_val,
group_names=labels,
categories=categories,
cbar=False,
title='Confusion Matrix: ' + alog_name,
figsize=(10, 10))
return scores, clf, cf_matrix_val
def model_training_out_of_time_tpot(df,
holdout,
target,
features,
show_holdout=False,
downsample=False):
X = df[features]
y = df[target].astype('bool')
X_holdout = holdout[features].drop('sec_filing_date', axis=1)
y_holdout = holdout[target].astype('bool')
scores = {'acc': [], 'f1': []}
cf_matrix_val = np.zeros((2, 2), dtype=np.int)
tbcv = TimeBasedCV(train_period=3, test_period=1, freq='years')
tbcv_folds = tbcv.split(df,
validation_split_date=datetime.date(2008, 12, 31),
date_column='sec_filing_date')
for train_index, test_index in tbcv_folds:
if downsample:
df_kfold = down_sample_majority_class(df, 'rating_downgrade')
df_kfold_index = df_kfold.index.tolist()
train_index = [
idx for idx in list(train_index) if idx in df_kfold_index
]
data_train = X.loc[train_index].drop('sec_filing_date', axis=1)
target_train = y.loc[train_index]
data_test = X.loc[test_index].drop('sec_filing_date', axis=1)
target_test = y.loc[test_index]
clf = TPOTClassifier(generations=5,
population_size=50,
verbosity=2,
max_time_mins=5)
clf.fit(data_train, target_train)
preds = clf.predict(data_test)
# accuracy for the current fold only
score = clf.score(data_test, target_test)
f1 = f1_score(target_test, preds)
cf_matrix_val += confusion_matrix(target_test, preds)
scores['acc'].append(score)
scores['f1'].append(f1)
print("Cross Validation Score: " +
str(sum(scores['acc']) / len(scores['acc'])))
if show_holdout:
# Test model trained on last three years on holdout data
frames = [test_index for train_index, test_index in tbcv_folds[-3:]]
frames = [item for sublist in frames for item in sublist]
data_train = X.loc[frames].drop('sec_filing_date', axis=1)
target_train = y.loc[frames]
clf.fit(data_train, target_train)
holdout_preds = clf.predict(X_holdout)
cf_matrix = confusion_matrix(y_holdout, holdout_preds)
print("Holdout Score: " + str(clf.score(X_holdout, y_holdout)))
print('\n')
# Visualize confusion matrix for holdout data
labels = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
categories = ['No Downgrade', 'Downgrade']
make_confusion_matrix(cf_matrix,
group_names=labels,
categories=categories,
cbar=False,
title='Confusion Matrix: TPOT',
figsize=(10, 10))
else:
#Visualize confusion matrix for cross-val data
labels = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
categories = ['No Downgrade', 'Downgrade']
make_confusion_matrix(cf_matrix_val,
group_names=labels,
categories=categories,
cbar=False,
title='Confusion Matrix: TPOT',
figsize=(10, 10))
return scores, clf, cf_matrix_val
def model_training_out_of_time_pretrained(df,
holdout,
target,
features,
show_holdout=False,
downsample=False):
"""
Title: Learning Word Embeddings from 10-K Filings for Financial NLP Tasks
Author: Saurabh Sehrawat
Date: 2019
Code version: 1.0
Availability: https://github.com/ssehrawat/10K-word-embeddings
"""
embed = torch.load(main_dir +
'data/10K-word-embeddings/10k_word_embeddings.tar')
vocab_to_int = torch.load(main_dir +
'data/10K-word-embeddings/vocab_to_int.tar')
X = df[features]
y = df[target].astype('bool')
X_holdout = holdout[features].drop('sec_filing_date', axis=1)
y_holdout = holdout[target].astype('bool')
alog_name = 'CNN'
print('### ' + alog_name + ' ###')
scores = dict()
cf_matrix_val = np.zeros((2, 2), dtype=np.int)
tbcv = TimeBasedCV(train_period=3, test_period=1, freq='years')
tbcv_folds = tbcv.split(df,
validation_split_date=datetime.date(2008, 12, 31),
date_column='sec_filing_date')
k_folds = len(tbcv_folds)
for k_index, (train_index, test_index) in enumerate(tbcv_folds):
if downsample:
df_kfold = down_sample_majority_class(df, 'rating_downgrade')
df_kfold_index = df_kfold.index.tolist()
train_index = [
idx for idx in list(train_index) if idx in df_kfold_index
]
data_train = X.loc[train_index].drop('sec_filing_date', axis=1)
target_train = y.loc[train_index]
data_test = X.loc[test_index].drop('sec_filing_date', axis=1)
target_test = y.loc[test_index]
print("=========================================")
print("==== K Fold Validation step => %d/%d ======" %
(k_index + 1, k_folds))
print("=========================================")
x_train, y_train, x_val, y_val, embedding_layer, MAX_SEQUENCE_LENGTH = create_embedding_layer(
data_train,
target_train,
data_test,
target_test,
embed,
vocab_to_int,
trainable=False)
history, model = train_model_keras_CNN(x_train, y_train, x_val, y_val,
embedding_layer,
MAX_SEQUENCE_LENGTH)
print(history.history)
scores[k_index] = history.history
preds_y = model.predict(x_val)
preds_y = np.rint(preds_y)
preds_y = preds_y.argmax(axis=-1)
y_val = y_val.argmax(axis=-1)
cf_matrix_val += confusion_matrix(y_val, preds_y)
if show_holdout:
# Test model trained on last three years on holdout data
frames = [test_index for train_index, test_index in tbcv_folds[-3:]]
frames = [item for sublist in frames for item in sublist]
data_train = X.loc[frames].drop('sec_filing_date', axis=1)
target_train = y.loc[frames]
x_train, y_train, x_val, y_val, embedding_layer, MAX_SEQUENCE_LENGTH = create_embedding_layer(
data_train,
target_train,
X_holdout,
y_holdout,
embed,
vocab_to_int,
trainable=False)
history, model = train_model_keras_CNN(x_train, y_train, x_val, y_val,
embedding_layer,
MAX_SEQUENCE_LENGTH)
preds_y = model.predict(x_val)
preds_y = np.rint(preds_y)
preds_y = preds_y.argmax(axis=-1)
y_val = y_val.argmax(axis=-1)
cf_matrix = confusion_matrix(y_val, preds_y)
scores['holdout'] = history.history
#print("Holdout Score: " + str(clf.score(x_val, y_val)))
#print('\n')
# Visualize confusion matrix for holdout data
labels = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
categories = ['No Downgrade', 'Downgrade']
make_confusion_matrix(cf_matrix,
group_names=labels,
categories=categories,
cbar=False,
title='Confusion Matrix: ' + alog_name,
figsize=(10, 10))
else:
labels = ['True Neg', 'False Pos', 'False Neg', 'True Pos']
categories = ['No Downgrade', 'Downgrade']
make_confusion_matrix(cf_matrix_val,
group_names=labels,
categories=categories,
cbar=False,
title='Confusion Matrix: ' + alog_name,
figsize=(10, 10))
return scores, cf_matrix_val