def synonym_relation(text1, text2): try: if (text1 == 'no info' or text2 == 'no info'): return 0 else: text1 = stopwordsremove(text1) text2 = stopwordsremove(text2) syn_set = set() count = 0 if (len(text1) == 0 or len(text2) == 0): return 0 if (len(text1) < len(text2)): for word in text2: for syn in wordnet.synsets(word): for l in syn.lemmas(): syn_set.add(l.name()) for word in text1: if word in syn_set: count += 1 return (count / len(text1)) else: for word in text1: for syn in wordnet.synsets(word): for l in syn.lemmas(): syn_set.add(l.name()) for word in text2: if word in syn_set: count += 1 return (count / len(text2)) except Exception as e: handle('synonym relation finding process')
def missing_values(data): """ All columns might contain missing values we have to decide how to handle each column. Some columns need to handled seperately. """ """ 1.location a second level of handling is done as of to remove numeric values in the location data. regex is used to remove those numeric data and replace with no info. """ print('Handling Missing Data') try: data['location'].fillna('no info', inplace=True) withoutcomma = data[~data['location'].str.contains(",")].index withcomma = data[data['location'].str.contains(",")].index for i in withcomma: data.loc[i, 'country'] = data.loc[i, 'location'].split(',')[0].strip() for i in withoutcomma: data.loc[i, 'country'] = data.loc[i, 'location'].strip() """2.salary range""" data['salary_range'].fillna('0-0', inplace=True) for i in range(0, data.shape[0]): str = data.loc[i, 'salary_range'] if re.search(r'[a-z,A-Z]', str): data.loc[i, 'salary_range'] = '0-0' if (data.loc[i, 'salary_range'].find("-") != -1): data.loc[i, 'minimum_salary'] = data.loc[ i, 'salary_range'].split('-')[0] data.loc[i, 'maximum_salary'] = data.loc[ i, 'salary_range'].split('-')[1] else: data.loc[i, 'minimum_salary'] = data.loc[i, 'salary_range'] data.loc[i, 'maximum_salary'] = data.loc[i, 'salary_range'] """3. All other categorical columns and remaining numeric columns.""" columns = data.columns for i in columns: if (data[i].isna().any()): if (data[i].dtypes == 'object'): data[i].fillna('no info', inplace=True) data[i] = data[i].str.lower() else: data[i].fillna(0, inplace=True) data.drop(['salary_range', 'location'], axis=1, inplace=True) return data except Exception as e: handle('missing data handling process')
def categorical_cols_test(data): print('Categorical Encoding') try: encoder = pickle.load(open("model/encoder.p", "rb")) newdata = encoder.transform(data) return newdata except Exception as e: handle('categorical columns handling for testing process')
def training(): try: data = read_csv('data/train.csv') (data.pipe(missing_values).pipe(texthandling) .pipe(categorical_cols_train).pipe(train_and_save_model)) except Exception as e: handle("Training piepline")
def testing(): try: data = read_csv('data/test.csv') (data.pipe(missing_values).pipe(texthandling).pipe( categorical_cols_test).pipe(load_model_predict)) except Exception as e: handle('testing process')
def read_csv(path): try: if ('csv' == path.split(".")[-1]): data = pd.read_csv(path) else: print("The files is not a CSV file") except Exception as e: handle('file reading') return data
def stopwordsremove(text): try: word_token = word_tokenize(text) ps = PorterStemmer() filtered = [ ps.stem(w.lower()) for w in word_token if not w in stop_words ] return filtered except Exception as e: handle('stop words removing')
def categorical_cols_train(data): try: print('Categorical Encoding') encoder = ce.BinaryEncoder(cols=[ 'employment_type', 'required_experience', 'required_education', 'country' ]) newdata = encoder.fit_transform(data) pickle.dump(encoder, open("model/encoder.p", "wb")) return newdata except Exception as e: handle('categorical column handling')
def load_model_predict(data): try: X_test = data.drop('fraudulent', axis=1) y_test = data['fraudulent'] scaler = pickle.load(open("model/scaler.p", "rb")) X_test = scaler.transform(X_test) filename = 'model/finalized_model.p' model = pickle.load(open(filename, 'rb')) y_pred = model.predict(X_test) score_and_save(y_pred) except Exception as e: handle('prediction process')
def removeuncessary(text): try: ''' 1. removing punctuations, 2. removing numbered words, 3. removing unknown characters ''' text = re.sub('[%s]' % re.escape(string.punctuation), '', str(text)) text = re.sub('\w*\d\w*', '', str(text)) text = re.sub('[^a-zA-Z ]+', ' ', str(text)) return text except Exception as e: handle('removing unnecessary text')
def score_and_save(y_pred): try: data = read_csv('data/test.csv') y_test = data['fraudulent'] cm = confusion_matrix(y_test, y_pred) print("\n" + "SCORES") print("confusion matrix") print(cm) print('F1-Score' + ' = ' + str(round(f1_score(y_test, y_pred), 4))) print('Precision' + ' = ' + str(round(precision_score(y_test, y_pred), 4))) print('Recall' + ' = ' + str(round(recall_score(y_test, y_pred), 4))) print('Accuracy' + ' = ' + str(round(accuracy_score(y_test, y_pred), 4))) data['fraud_prediction'] = y_pred data.to_csv('predictionoutput/testsetprediction.csv') except Exception as e: handle('scoring and saving process')
def train_and_save_model(data): try: print("Model Training") X_train = data.drop('fraudulent', axis=1) y_train = data['fraudulent'] sc = StandardScaler() X_train = sc.fit_transform(X_train) pickle.dump(sc, open("model/scaler.p", "wb")) from sklearn.ensemble import RandomForestClassifier model = RandomForestClassifier(n_estimators=100, criterion='entropy', random_state=1) model.fit(X_train, y_train) filename = 'model/finalized_model.p' pickle.dump(model, open(filename, 'wb')) except Exception as e: handle('Model Creation and training')
def texthandling(data): print('Text Handling') try: ''' This function is for handling text data columns company profile, description, requirements, benefits are there is multiple text in those columns we need to do something about them. ''' stop_words = set(stopwords.words('english')) for i in range(0, data.shape[0]): data.loc[i, 'company_profile'] = removeuncessary( data.loc[i, 'company_profile']) data.loc[i, 'description'] = removeuncessary(data.loc[i, 'description']) data.loc[i, 'requirements'] = removeuncessary( data.loc[i, 'requirements']) data.loc[i, 'benefits'] = removeuncessary(data.loc[i, 'benefits']) data.loc[i, 'title'] = removeuncessary(data.loc[i, 'title']) data.loc[i, 'department'] = removeuncessary(data.loc[i, 'department']) data.loc[i, 'industry'] = removeuncessary(data.loc[i, 'industry']) data.loc[i, 'function'] = removeuncessary(data.loc[i, 'function']) words = str(data.loc[i, 'company_profile']) if (words == 'no info'): data.loc[i, 'company_profile_word_count'] = 0 else: data.loc[i, 'company_profile_word_count'] = len(words.split()) words = str(data.loc[i, 'benefits']) if (words == 'no info'): data.loc[i, 'benefits_word_count'] = 0 else: data.loc[i, 'benefits_word_count'] = len(words.split()) data.loc[i, 'title_and_job_similarity'] = synonym_relation( data.loc[i, 'title'], data.loc[i, 'description']) data.loc[i, 'title_and_req_similarity'] = synonym_relation( data.loc[i, 'title'], data.loc[i, 'requirements']) data.loc[i, 'profile_and_job_similarity'] = synonym_relation( data.loc[i, 'company_profile'], data.loc[i, 'description']) data.loc[i, 'profiel_and_req_similarity'] = synonym_relation( data.loc[i, 'company_profile'], data.loc[i, 'requirements']) data.loc[i, 'title_and_department_syn_similarity'] = synonym_relation( data.loc[i, 'title'], data.loc[i, 'department']) data.loc[i, 'title_and_industry_syn_similarity'] = synonym_relation( data.loc[i, 'title'], data.loc[i, 'industry']) data.loc[i, 'title_and_function_syn_similarity'] = synonym_relation( data.loc[i, 'title'], data.loc[i, 'function']) data.loc[ i, 'industry_and_department_syn_similarity'] = synonym_relation( data.loc[i, 'industry'], data.loc[i, 'department']) data.loc[ i, 'function_and_department_syn_similarity'] = synonym_relation( data.loc[i, 'function'], data.loc[i, 'department']) data.loc[ i, 'industry_and_function_syn_similarity'] = synonym_relation( data.loc[i, 'industry'], data.loc[i, 'function']) for i in [ 'title_and_job_similarity', 'title_and_req_similarity', 'profile_and_job_similarity', 'profiel_and_req_similarity', 'title_and_department_syn_similarity', 'title_and_industry_syn_similarity', 'title_and_function_syn_similarity', 'function_and_department_syn_similarity', 'industry_and_department_syn_similarity', 'industry_and_function_syn_similarity' ]: data[i].fillna(0, inplace=True) data.drop([ 'company_profile', 'benefits', 'description', 'requirements', 'title', 'department', 'industry', 'function', 'job_id' ], axis=1, inplace=True) return data except Exception as e: handle('Text handling process')
import argparse from Allcodefiles.training import training from Allcodefiles.testing import testing from Allcodefiles.Exceptionhandling import handle if __name__ == '__main__': parser = argparse.ArgumentParser(description = 'Fake Job Prediction') parser.add_argument('-r', '--return_object', choices = ['train', 'test'], default = 'train', type = str, help = 'Select what task to be done') args = parser.parse_args() var_args = vars(args) try: if(var_args['return_object'] == 'train'): training() else: testing() except Exception as e: handle('Main file')