def unionx(spdr):
data_path = spdr.DATA_PATH
mt_file = 'X.npz', 'Y.npz'
ft_order = ['lem', 'nn', 'ner', 'parse', 'vc', 'mesh', 'chem', 'extmesh']
import bionlp.util.io as io
X, Y = io.read_df(os.path.join(data_path, mt_file[0]), with_idx=True, sparse_fmt='csr'), io.read_df(os.path.join(data_path, mt_file[1]), with_idx=True, sparse_fmt='csr')
ft_idx = {}
for i, col in enumerate(X.columns):
for ft in ft_order:
if (col.startswith(ft+'_')):
ft_idx.setdefault(ft, []).append(i)
break
union_ft_idx = {}
for j in range(Y.shape[1]):
X_j = io.read_df(os.path.join(data_path, 'X_%i.npz'%j), with_idx=True, sparse_fmt='csr')
for i, col in enumerate(X_j.columns):
for ft in ft_order:
if (col.startswith(ft+'_')):
union_ft_idx.setdefault(ft, set([])).add(col)
break
new_ft = []
for ft in ft_order:
new_ft.extend(list(union_ft_idx[ft]))
union_X = X.loc[:,new_ft]
io.write_df(union_X, os.path.join(data_path, 'union_X.npz'), with_idx=True, sparse_fmt='csr', compress=True)
def decomp_data(method='LDA', n_components=100): X, Y = spdr.get_data(None, ft_type=opts.type, max_df=ast.literal_eval(opts.maxdf), min_df=ast.literal_eval(opts.mindf), from_file=True, fmt=opts.fmt, spfmt=opts.spfmt) method = method.upper() n_components = min(n_components, X.shape[1]) if (method == 'LDA'): model = make_pipeline(LatentDirichletAllocation(n_topics=n_components, learning_method='online', learning_offset=50., max_iter=5, n_jobs=opts.np, random_state=0), Normalizer(copy=False)) elif (method == 'NMF'): model = make_pipeline(NMF(n_components=n_components, random_state=0, alpha=.1, l1_ratio=.5), Normalizer(copy=False)) elif (method == 'LSI'): model = make_pipeline(TruncatedSVD(n_components), Normalizer(copy=False)) elif (method == 'TSNE'): model = make_pipeline(ftdecomp.DecompTransformer(n_components, ftdecomp.t_sne, initial_dims=15*n_components, perplexity=30.0)) if (opts.prefix == 'all'): td_cols = X.columns else: # Only apply dimension reduction on specific columns td_cols = np.array(map(lambda x: True if any(x.startswith(prefix) for prefix in opts.prefix.split(SC)) else False, X.columns)) td_X = X.loc[:,td_cols] new_td_X = model.fit_transform(td_X.as_matrix()) if (opts.prefix == 'all'): columns = range(new_td_X.shape[1]) if not hasattr(model.steps[0][1], 'components_') else td_X.columns[model.steps[0][1].components_.argmax(axis=1)] new_X = pd.DataFrame(new_td_X, index=X.index, columns=['tp_%s' % x for x in columns]) else: columns = range(new_td_X.shape[1]) if not hasattr(model.steps[0][1], 'components_') else td_X.columns[model.steps[0][1].components_.argmax(axis=1)] # Concatenate the components and the columns are not applied dimension reduction on new_X = pd.concat([pd.DataFrame(new_td_X, index=X.index, columns=['tp_%s' % x for x in columns]), X.loc[:,np.logical_not(td_cols)]], axis=1) if (opts.fmt == 'npz'): io.write_df(new_X, os.path.join(spdr.DATA_PATH, '%s%i_X.npz' % (method.lower(), n_components)), with_idx=True, sparse_fmt=opts.spfmt, compress=True) else: new_X.to_csv(os.path.join(spdr.DATA_PATH, '%s%i_X.csv' % (method.lower(), n_components)), encoding='utf8')
def filtx(spdr): data_path = spdr.DATA_PATH mt_file = 'X.npz', 'Y.npz' import bionlp.util.io as io X, Y = io.read_df(os.path.join(data_path, mt_file[0]), with_idx=True, sparse_fmt='csr'), io.read_df(os.path.join(data_path, mt_file[1]), with_idx=True, sparse_fmt='csr') Xs = spdr.ft_filter(X, Y) for i, x_df in enumerate(Xs): io.write_df(x_df, os.path.join(data_path, 'X_%i.npz' % i), with_idx=True, sparse_fmt='csr', compress=True)
def extend_mesh(ft_type='binary'): X, Y = spdr.get_data(None, ft_type=opts.type, max_df=ast.literal_eval(opts.maxdf), min_df=ast.literal_eval(opts.mindf), from_file=True, fmt=opts.fmt, spfmt=opts.spfmt) mesh_df = mm.mesh_countvec(X.index) mesh_df.columns = ['extmesh_' + x for x in mesh_df.columns] new_X = pd.concat([X, mesh_df], axis=1, join_axes=[X.index]) print 'The size of data has been changed from %s to %s.' % (X.shape, new_X.shape) if (opts.fmt == 'npz'): io.write_df(new_X, os.path.join(spdr.DATA_PATH, 'extmesh_X.npz'), with_idx=True, sparse_fmt=opts.spfmt, compress=True) else: new_X.to_csv(os.path.join(spdr.DATA_PATH, 'extmesh_X.csv'), encoding='utf8')
def gen_data(): if (opts.local): X, Y = spdr.get_data(None, from_file=True) else: pmid_list = spdr.get_pmids() articles = spdr.fetch_artcls(pmid_list) X, Y = spdr.get_data(articles, ft_type=opts.type, max_df=ast.literal_eval(opts.maxdf), min_df=ast.literal_eval(opts.mindf), fmt=opts.fmt, spfmt=opts.spfmt) hallmarks = Y.columns # Feature Selection # mt = sp.sparse.coo_matrix(X) # mask_mt = np.zeros(mt.shape) # mask_mt[mt.row, mt.col] = 1 # stat = mask_mt.sum(axis=0) # cln_X = X.iloc[:,np.arange(stat.shape[0])[stat>ast.literal_eval(opts.thrshd) * (stat.max() - stat.min()) + stat.min()]] # Document Frequence # stat, _ = ftslct.freqs(X.values, Y.values) # Mutual information # stat, _ = ftslct.mutual_info(X.values, Y.values) # Information gain # stat, _ = ftslct.info_gain(X.values, Y.values) # GSS coefficient # stat, _ = ftslct.gss_coef(X.values, Y.values) # NGL coefficient # stat, _ = ftslct.ngl_coef(X.values, Y.values) # Odds ratio # stat, _ = ftslct.odds_ratio(X.values, Y.values) # Fisher criterion # stat, _ = ftslct.fisher_crtrn(X.values, Y.values) # GU metric # stat, _ = ftslct.gu_metric(X.values, Y.values) # Decision tree # stat, _ = ftslct.decision_tree(X.values, Y.values) # Combined feature stat, _ = ftslct.utopk(X.values, Y.values, ftslct.decision_tree, fn=100) io.write_npz(stat, os.path.join(spdr.DATA_PATH, 'ftw.npz')) # cln_X = X.iloc[:,np.arange(stat.shape[0])[stat>stat.min()]] cln_X = X.iloc[:,stat.argsort()[-500:][::-1]] print 'The size of data has been changed from %s to %s.' % (X.shape, cln_X.shape) if (opts.fmt == 'npz'): io.write_df(cln_X, os.path.join(spdr.DATA_PATH, 'cln_X.npz'), with_idx=True, sparse_fmt=opts.spfmt, compress=True) else: cln_X.to_csv(os.path.join(spdr.DATA_PATH, 'cln_X.csv'), encoding='utf8') del X, cln_X for i in range(Y.shape[1]): y = Y.iloc[:,i] if (opts.fmt == 'npz'): io.write_df(y, os.path.join(spdr.DATA_PATH, 'y_%s.npz' % i), with_col=False, with_idx=True) else: y.to_csv(os.path.join(spdr.DATA_PATH, 'y_%s.csv' % i), encoding='utf8')
def avgfeatw(dir_path='.'):
df_list = []
for file in fs.listf(dir_path):
if file.endswith(".npz"):
df_list.append(io.read_df(os.path.join(dir_path, file), with_idx=True))
feat_w_mt = pd.concat([df.loc[:,'Importance Mean'] for df in df_list], axis=1, join_axes=[df_list[0].index]).astype('float').values
feat_w_avg = feat_w_mt.mean(axis=1)
feat_w_std = feat_w_mt.std(axis=1)
sorted_idx = np.argsort(feat_w_avg, axis=-1)[::-1]
sorted_feat_w = np.column_stack((df_list[0].loc[:,'Feature Name'].values[sorted_idx], feat_w_avg[sorted_idx], feat_w_std[sorted_idx]))
feat_w_df = pd.DataFrame(sorted_feat_w, index=df_list[0].index.values[sorted_idx], columns=['Feature Name', 'Importance Mean', 'Importance Std'])
feat_w_df.to_excel(os.path.join(dir_path, 'featw.xlsx'))
io.write_df(feat_w_df, os.path.join(dir_path, 'featw'), with_idx=True)
def add_d2v(n_components=100, win_size=8, min_t=5, mdl_fname='d2v.mdl'):
from gensim.parsing.preprocessing import preprocess_string
from gensim.models.doc2vec import TaggedDocument, Doc2Vec
def read_files(fpaths, code='ascii'):
for fpath in fpaths:
try:
yield TaggedDocument(words=preprocess_string('\n'.join(fs.read_file(fpath, code))), tags=[os.path.splitext(os.path.basename(fpath))[0]])
except Exception as e:
continue
def read_prcsed_files(fpaths, code='ascii'):
for fpath in fpaths:
try:
words = []
for line in fs.read_file(fpath, code):
if (line == '~~~'):
continue
if (line == '. . .' or line == '~~~ ~~~' or line == ', , ,'):
continue
items = line.split()
if (len(items) < 3): # Skip the unrecognized words
continue
words.append(items[2].lower())
yield TaggedDocument(words=words, tags=[os.path.splitext(os.path.basename(fpath))[0]])
except Exception as e:
continue
mdl_fpath = os.path.join(spdr.DATA_PATH, mdl_fname)
if (os.path.exists(mdl_fpath)):
model = Doc2Vec.load(mdl_fpath)
else:
# model = Doc2Vec(read_files(fs.listf(spdr.ABS_PATH, full_path=True)), size=n_components, window=8, min_count=5, workers=opts.np)
model = Doc2Vec(read_prcsed_files(fs.listf(os.path.join(spdr.DATA_PATH, 'lem'), full_path=True)), size=n_components, window=8, min_count=5, workers=opts.np)
model.save(os.path.join(spdr.DATA_PATH, mdl_fname))
X, Y = spdr.get_data(None, ft_type=opts.type, max_df=ast.literal_eval(opts.maxdf), min_df=ast.literal_eval(opts.mindf), from_file=True, fmt=opts.fmt, spfmt=opts.spfmt)
# Map the index of original matrix to that of the paragraph vectors
d2v_idx = [model.docvecs.index_to_doctag(i).rstrip('.lem') for i in range(model.docvecs.count)]
mms = MinMaxScaler()
d2v_X = pd.DataFrame(mms.fit_transform(model.docvecs[range(model.docvecs.count)]), index=d2v_idx, columns=['d2v_%i' % i for i in range(model.docvecs[0].shape[0])])
# d2v_X = pd.DataFrame(model.docvecs[range(model.docvecs.count)], index=d2v_idx, columns=['d2v_%i' % i for i in range(model.docvecs[0].shape[0])])
new_X = pd.concat([X, d2v_X], axis=1, join_axes=[X.index])
print 'The size of data has been changed from %s to %s.' % (X.shape, new_X.shape)
if (opts.fmt == 'npz'):
io.write_df(d2v_X, os.path.join(spdr.DATA_PATH, 'd2v_X.npz'), with_idx=True, sparse_fmt=opts.spfmt, compress=True)
io.write_df(new_X, os.path.join(spdr.DATA_PATH, 'cmb_d2v_X.npz'), with_idx=True, sparse_fmt=opts.spfmt, compress=True)
else:
d2v_X.to_csv(os.path.join(spdr.DATA_PATH, 'd2v_X.csv'), encoding='utf8')
new_X.to_csv(os.path.join(spdr.DATA_PATH, 'cmb_d2v_X.csv'), encoding='utf8')
def pred2cor(dir_path, file_ptn, mdls, pids=range(10), crsval=10):
import scipy.stats as stats
from chm_annot import pred_ovl
import bionlp.util.math as imath
for pid in pids:
crsval_povl, crsval_spearman = [[] for i in range(2)]
for crs_t in xrange(crsval):
preds, true_lb = [], None
for mdl in mdls:
mdl = mdl.replace(' ', '_').lower()
file = file_ptn.replace('#CRST#', str(crs_t)).replace('#MDL#', mdl).replace('#PID#', str(pid))
npz_file = io.read_npz(os.path.join(dir_path, file))
preds.append(npz_file['pred_lb'])
true_lb = npz_file['true_lb']
preds_mt = np.column_stack([x.ravel() for x in preds])
preds.append(true_lb)
tpreds_mt = np.column_stack([x.ravel() for x in preds])
crsval_povl.append(pred_ovl(preds_mt, true_lb.ravel()))
crsval_spearman.append(stats.spearmanr(tpreds_mt))
povl_avg = np.array(crsval_povl).mean(axis=0).round()
spmnr_avg = np.array([crsp[0] for crsp in crsval_spearman]).mean(axis=0)
spmnr_pval = np.array([crsp[1] for crsp in crsval_spearman]).mean(axis=0)
povl_idx = list(imath.subset(mdls, min_crdnl=1))
povl_avg_df = pd.DataFrame(povl_avg, index=povl_idx, columns=['pred_ovl', 'tpred_ovl'])
spmnr_avg_df = pd.DataFrame(spmnr_avg, index=mdls+['Annotations'], columns=mdls+['Annotations'])
spmnr_pval_df = pd.DataFrame(spmnr_pval, index=mdls+['Annotations'], columns=mdls+['Annotations'])
povl_avg_df.to_excel(os.path.join(dir_path, 'cpovl_avg_%s.xlsx' % pid))
spmnr_avg_df.to_excel(os.path.join(dir_path, 'spmnr_avg_%s.xlsx' % pid))
spmnr_pval_df.to_excel(os.path.join(dir_path, 'spmnr_pval_%s.xlsx' % pid))
io.write_df(povl_avg_df, os.path.join(dir_path, 'povl_avg_%s.npz' % pid), with_idx=True)
io.write_df(spmnr_avg_df, os.path.join(dir_path, 'spmnr_avg_%s.npz' % pid), with_idx=True)
io.write_df(spmnr_pval_df, os.path.join(dir_path, 'spmnr_val_%s.npz' % pid), with_idx=True)
def leave1out(spdr, mltl=True):
data_path = spdr.DATA_PATH
mt_file = 'X.npz', 'Y.npz'
ft_order = ['lem', 'nn', 'ner', 'parse', 'vc', 'mesh', 'chem']
import bionlp.util.io as io
X, Y = io.read_df(os.path.join(data_path, mt_file[0]), with_idx=True, sparse_fmt='csr'), io.read_df(os.path.join(data_path, mt_file[1]), with_idx=True, sparse_fmt='csr')
ft_dict = {}
for col in X.columns:
for ft in ft_order:
if (col.startswith(ft+'_')):
ft_dict.setdefault(ft, []).append(col)
break
if (mltl):
for ft in ft_order:
new_X = X.drop(ft_dict[ft], axis=1)
io.write_df(new_X, os.path.join(data_path, 'l1o_%s_X.npz'%ft), sparse_fmt='csr', compress=True)
else:
for i in range(Y.shape[1]):
X_i = io.read_df(os.path.join(data_path, 'X_%i.npz'%i), sparse_fmt='csr')
for ft in ft_order:
new_X = X_i.drop(ft_dict[ft], axis=1)
io.write_df(new_X, os.path.join(data_path, 'l1o_%s_X_%i.npz'%(ft,i)), sparse_fmt='csr', compress=True)
def expand_data(ft_type='binary', db_name='mesh2016', db_type='LevelDB', store_path='store'):
from rdflib import Graph
from bionlp.util import ontology
X, Y = spdr.get_data(None, ft_type=opts.type, max_df=ast.literal_eval(opts.maxdf), min_df=ast.literal_eval(opts.mindf), from_file=True, fmt=opts.fmt, spfmt=opts.spfmt)
mesh_cols = filter(lambda x: x.startswith('mesh_') or x.startswith('extmesh_'), X.columns)
mesh_X = X.loc[:,mesh_cols]
exp_meshx = set([])
ext_meshx_dict = {}
g = Graph(store=db_type, identifier=db_name)
g.open(store_path)
for col in mesh_X.columns:
mesh_lb = col.strip('extmesh_').strip('mesh_').replace('"', '\\"')
# Get similar MeSH terms
em_set = set(ontology.slct_sim_terms(g, mesh_lb, prdns=[('meshv',ontology.MESHV)], eqprds=ontology.MESH_EQPRDC_MAP))
# Overall extended MeSH terms
exp_meshx |= em_set
# Extended MeSH terms per column
ext_meshx_dict[col] = em_set
g.close()
exp_mesh_X = pd.DataFrame(np.zeros((mesh_X.shape[0], len(exp_meshx)), dtype='int8'), index=X.index, columns=['expmesh_%s' % w for w in exp_meshx])
# Append the similar MeSH terms of each column to the final matrix
for col, sim_mesh in ext_meshx_dict.iteritems():
if (len(sim_mesh) == 0): continue
sim_cols = ['expmesh_%s' % w for w in sim_mesh]
if (ft_type == 'binary'):
exp_mesh_X.loc[:,sim_cols] = np.logical_or(exp_mesh_X.loc[:,sim_cols], mesh_X.loc[:,col].reshape((-1,1))).astype('int')
elif (ft_type == 'numeric'):
exp_mesh_X.loc[:,sim_cols] += mesh_X.loc[:,col].reshape((-1,1))
elif (ft_type == 'tfidf'):
pass
new_X = pd.concat([X, exp_mesh_X], axis=1, join_axes=[X.index])
print 'The size of data has been changed from %s to %s.' % (X.shape, new_X.shape)
if (opts.fmt == 'npz'):
io.write_df(new_X, os.path.join(spdr.DATA_PATH, 'exp_X.npz'), with_idx=True, sparse_fmt=opts.spfmt, compress=True)
else:
new_X.to_csv(os.path.join(spdr.DATA_PATH, 'exp_X.csv'), encoding='utf8')
def _pred2event(spdr_mod,
combined,
pred_fpath,
data_path,
test_X_paths=['cbow/dev_X%i' % i for i in range(4)],
train_Y_path='cbow/train_Y',
method='cbow',
source='2011',
task='bgi'):
pred = io.read_npz(pred_fpath)['pred_lb']
if (combined):
event_mt = np.column_stack(
[pred[:, i] for i in range(0, pred.shape[1], 2)])
dir_mt = np.column_stack(
[pred[:, i] for i in range(1, pred.shape[1], 2)])
else:
evnt_num = pred.shape[1] / 2
event_mt = pred[:, :evnt_num]
dir_mt = pred[:, evnt_num:]
np.place(dir_mt, dir_mt == 0, [-1])
event_mt *= dir_mt
test_Xs = [pd.read_hdf(data_path, dspath) for dspath in test_X_paths]
train_Y = pd.read_hdf(data_path, train_Y_path)
test_Y = pd.DataFrame(event_mt,
index=test_Xs[0].index,
columns=train_Y.columns)
io.write_df(test_Y,
'test_Y',
with_idx=True,
sparse_fmt='csr',
compress=True)
events = spdr_mod.pred2data(test_Y,
method=method,
source=source,
task=task)
spdr_mod.to_a2(events, './pred', source=source, task=task)
def get_data(articles, from_file=None, ft_type='binary', max_df=1.0, min_df=1, fmt='npz', spfmt='csr'):
# Read from local files
if (from_file):
if (type(from_file) == bool):
file_name = 'X.npz' if (fmt == 'npz') else 'X.csv'
else:
file_name = from_file
print 'Reading file: %s and Y.%s' % (file_name, fmt)
if (fmt == 'npz'):
return io.read_df(os.path.join(DATA_PATH, file_name), with_idx=True, sparse_fmt=spfmt), io.read_df(os.path.join(DATA_PATH, 'Y.npz'), with_idx=True, sparse_fmt=spfmt)
else:
return pd.read_csv(os.path.join(DATA_PATH, file_name), index_col=0, encoding='utf8'), pd.read_csv(os.path.join(DATA_PATH, 'Y.csv'), index_col=0, encoding='utf8')
## Feature columns
ft_pmid, ft_abs, ft_lem, ft_nnv, ft_ner, ft_parse, ft_vc, ft_mesh, ft_chem, label = [[] for i in range(10)]
ft_order = ['lem', 'nn', 'ner', 'parse', 'vc', 'mesh', 'chem']
ft_name = {'lem':'LBoW', 'nn':'N-Bigrams', 'ner':'NE', 'parse':'GR', 'vc':'VC', 'mesh':'MeSH', 'chem':'Chem'}
ft_dic = {'lem':ft_lem, 'ner':ft_ner, 'parse':ft_parse, 'vc':ft_vc, 'mesh':ft_mesh, 'chem':ft_chem}
hm_lb = ['PS', 'GS', 'CD', 'RI', 'A', 'IM', 'GI', 'TPI', 'CE', 'ID']
bft_dic, hm_stat = [{} for i in range(2)]
label_set = set()
# sent_splitter = nltk.data.load('tokenizers/punkt/english.pickle')
for artcl in articles:
ft_pmid.append(artcl['id'])
ft_abs.append(artcl['abs'])
ft_mesh.append(artcl['mesh'])
ft_chem.append(artcl['chem'])
label.append(artcl['annots'])
label_set.update(artcl['annots'])
c = Counter(artcl['annots'])
for hm, num in c.iteritems():
hs = hm_stat.setdefault(hm, [0,0])
hs[0], hs[1] = hs[0] + 1, hs[1] + num
# hs[0], hs[1] = hs[0] + 1, hs[1] + len(sent_splitter.tokenize(artcl['abs'].strip()))
# uniq_lb = list(label_set)
uniq_lb = [IHM_MAP[lb] for lb in hm_lb]
## Get the feature sets of the specific hallmark
# feat_sets = get_fsnames()
feat_sets = ft_order
feature_sets, feat_stat = get_featsets(feat_sets, len(uniq_lb))
ft_stat_mt = np.array([feat_stat[ft] for ft in ft_order]).T
ft_stat_pd = pd.DataFrame(ft_stat_mt, index=hm_lb, columns=[ft_name[fset] for fset in feat_sets])
hm_stat_pd = pd.DataFrame([hm_stat[lb] for lb in uniq_lb], index=hm_lb, columns=['No. abstracts', 'No. sentences'])
if (fmt == 'npz'):
io.write_df(ft_stat_pd, os.path.join(DATA_PATH, 'ft_stat.npz'))
io.write_df(hm_stat_pd, os.path.join(DATA_PATH, 'hm_stat.npz'), with_idx=True)
else:
ft_stat_pd.to_csv(os.path.join(DATA_PATH, 'ft_stat.csv'), encoding='utf8')
hm_stat_pd.to_csv(os.path.join(DATA_PATH, 'hm_stat.csv'), encoding='utf8')
## Extract the features from the preprocessed data
for i in range(len(feat_sets)):
fset = feat_sets[i]
feature_set = feature_sets[i]
if (fset == 'chem' or fset == 'mesh'):
continue
if (fset == 'nn'):
continue
for pmid in ft_pmid:
feature, extra_feat = [[], []]
prev_term = ''
for line in fs.read_file(os.path.join(DATA_PATH, fset, '.'.join([pmid, fset, 'txt'])), 'utf8'):
if (line == '~~~'):
continue
if (fset == 'lem'):
if (line == '. . .' or line == '~~~ ~~~' or line == ', , ,'):
continue
items = line.split()
if (len(items) < 3): # Skip the unrecognized words
continue
feature.append(items[2].lower())
# Extract NN feature
if (items[1] == 'NN'):
if (prev_term != ''):
extra_feat.append(prev_term + ' ' + items[0].lower())
prev_term = items[0].lower()
else:
prev_term = ''
if (fset == 'ner'):
feature.append(line)
if (fset == 'parse'):
record = line.strip('()').replace(' _ ', ' ').split()
feature.append(','.join([w.split('_')[0] for w in record]).lower())
if (fset == 'vc'):
feature.extend(line.split())
ft_dic[fset].append(feature)
if (fset == 'lem'):
ft_nnv.extend(extra_feat)
## Convert the raw features into binary features
ft_type = ft_type.lower()
for i in range(len(feat_sets)):
fset = feat_sets[i]
feature_set = feature_sets[i]
if (fset == 'nn'):
bigram_vectorizer = CountVectorizer(ngram_range=(2, 2), token_pattern=r'\b\w+\b', max_df=max_df, min_df=min_df, vocabulary=set(ft_nnv), binary=True if ft_type=='binary' else False)
ft_nn = bigram_vectorizer.fit_transform(ft_abs).tocsr()
nn_classes = [cls[0] for cls in sorted(bigram_vectorizer.vocabulary_.items(), key=operator.itemgetter(1))]
bft_dic[fset] = (ft_nn, nn_classes)
continue
# overall_ft = list(set([ft for samp in ft_dic[fset] for ft in samp if ft]))
# mlb = MultiLabelBinarizer(classes=overall_ft)
# bft_dic[fset] = (mlb.fit_transform(ft_dic[fset]), mlb.classes_)
count_vectorizer = CountVectorizer(tokenizer=lambda text: [t for t in text.split('*#@') if t and t not in string.punctuation], lowercase=False, stop_words='english', token_pattern=r'\b\w+\b', max_df=max_df, min_df=min_df, binary=True if ft_type=='binary' else False)
ft_all = count_vectorizer.fit_transform(['*#@'.join(samp) for samp in ft_dic[fset]])
all_classes = [cls[0] for cls in sorted(count_vectorizer.vocabulary_.items(), key=operator.itemgetter(1))]
bft_dic[fset] = (ft_all, all_classes)
## Convert the annotations of each document to binary labels
mlb = MultiLabelBinarizer(classes=uniq_lb)
bin_label = (mlb.fit_transform(label), mlb.classes_)
## Generate the features as well as the labels to form a completed dataset
feat_mt = sp.sparse.hstack([bft_dic[fset][0] for fset in ft_order])
if (ft_type == 'tfidf'):
transformer = TfidfTransformer(norm='l2', sublinear_tf=False)
feat_mt = transformer.fit_transform(feat_mt)
feat_cols = ['%s_%s' % (fset, w) for fset in ft_order for w in bft_dic[fset][1]]
feat_df = pd.DataFrame(feat_mt.todense(), index=ft_pmid, columns=feat_cols)
label_df = pd.DataFrame(bin_label[0], index=ft_pmid, columns=bin_label[1])
obj_samp_idx = np.random.random_integers(0, feat_df.shape[0] - 1, size=200).tolist()
ft_samp_idx = np.random.random_integers(0, feat_df.shape[1] - 1, size=1000).tolist()
samp_feat_df = feat_df.iloc[obj_samp_idx, ft_samp_idx]
samp_lb_df = label_df.iloc[obj_samp_idx,:]
if (fmt == 'npz'):
io.write_df(feat_df, os.path.join(DATA_PATH, 'X.npz'), with_idx=True, sparse_fmt=spfmt, compress=True)
io.write_df(label_df, os.path.join(DATA_PATH, 'Y.npz'), with_idx=True, sparse_fmt=spfmt, compress=True)
io.write_df(samp_feat_df, os.path.join(DATA_PATH, 'sample_X.npz'), with_idx=True, sparse_fmt=spfmt, compress=True)
io.write_df(samp_lb_df, os.path.join(DATA_PATH, 'sample_Y.npz'), with_idx=True, sparse_fmt=spfmt, compress=True)
else:
feat_df.to_csv(os.path.join(DATA_PATH, 'X.csv'), encoding='utf8')
label_df.to_csv(os.path.join(DATA_PATH, 'Y.csv'), encoding='utf8')
samp_feat_df.to_csv(os.path.join(DATA_PATH, 'sample_X.csv'), encoding='utf8')
samp_lb_df.to_csv(os.path.join(DATA_PATH, 'sample_Y.csv'), encoding='utf8')
return feat_df, label_df
