def paper_single_process(params=None):
(df, i) = params
print(i, 'start', datetime.now())
df['titl'] = df['title'].fillna('').progress_apply(lambda s: preprocess(s))
df['abst'] = df['abstract'].fillna('').progress_apply(
lambda s: preprocess(s))
print(i, 'completed', datetime.now())
return df
def tr_single_process(params=None):
(tr, i) = params
print(i, 'start', datetime.now())
tr['quer_key'] = tr['description_text'].fillna('').progress_apply(
lambda s: preprocess(digest(s)))
tr['quer_all'] = tr['description_text'].fillna('').progress_apply(
lambda s: preprocess(s))
print(i, 'completed', datetime.now())
return tr
def transform(self, X, y=None):
return nlp_preprocess.preprocess(X, self.text_options)