def train(row_id_str, ds_id, hdfs_feat_dir, local_out_dir, ml_opts_jstr
, sp_master, spark_rdd_compress, spark_driver_maxResultSize, sp_exe_memory, sp_core_max
, zipout_dir, zipcode_dir, zip_file_name
, mongo_tuples, labelnameflag, fromweb, src_filename
, jobname ):
# create zip files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir, zipcode_dir, zip_file_name, prefix='zip_feature_util')
print "INFO: zip_file_path=",zip_file_path
#data_folder = hdfs_feat_dir + "/"
#local_out_dir = local_out_dir + "/"
#if os.path.exists(local_out_dir):
# shutil.rmtree(local_out_dir) # to keep smaplelist file
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
# init Spark context ====
sc=ml_util.ml_get_spark_context(sp_master
, spark_rdd_compress
, spark_driver_maxResultSize
, sp_exe_memory
, sp_core_max
, jobname
, [zip_file_path])
# start here =================================================================== ===============
t0 = time()
### Need to check if PCA available here ===========================
libsvm_data_file = os.path.join(hdfs_feat_dir , src_filename) # need to set k numb in filename somehow
print "INFO: libsvm_data_file=", libsvm_data_file
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file).cache()
# load sample RDD from text file
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
feature_count=0
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(sc, libsvm_data_file, feature_count, '')
# get label as a list
labels_list_all = samples_rdd.map(lambda p: int(p[0].label)).collect()
total_sample_count=len(labels_list_all)
parsedData =samples_rdd.map(lambda p: p[0].features).cache()
#for i in parsedData.collect(): #p.features: pyspark.mllib.linalg.SparseVector
# print "pd=",type(i),",i=",i
t1 = time()
print 'INFO: running time: %f' %(t1-t0)
t0 = t1
###############################################
########## build learning model ###############
###############################################
### get the parameters###
print "INFO: ============Learning Algorithm and Parameters============="
para_dict = json.loads(ml_opts_jstr)
flag_model = para_dict['learning_algorithm'] # kmeans
iteration_num = eval(para_dict['iterations'])
k=2
if 'k' in para_dict:
k = eval(para_dict['k'])
print "INFO: Learning Algorithm:", flag_model
print "INFO: iterations=", iteration_num
#print "training_sample_number=", training_sample_number
### generate label names (family names) #####
### connect to database to get the column list which contains all column number of the corresponding feature####
if labelnameflag == 1:
key = "dic_name_label"
jstr_filter='{"rid":'+row_id_str+',"key":"'+key+'"}'
jstr_proj='{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter='{"rid":'+ds_id+',"key":"'+key+'"}'
doc=query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_list = doc['value']
label_dic = {}
for i in range(0, len(dic_list)):
for key in dic_list[i]:
label_dic[dic_list[i][key]] = key.encode('UTF8')
print "INFO: label_dic:", label_dic
else:
label_dic = {}
label_set = set(labels_list_all)
for label_value in label_set:
label_dic[int(label_value)] = str(int(label_value))
print "INFO: generated label_dic:", label_dic
labels_list = []
for key in sorted(label_dic):
labels_list.append(label_dic[key])
print "INFO: labels_list=", labels_list
### build model ###
if flag_model == "kmeans":
print "=================== Kmeans ============"
model = KMeans.train(parsedData, k, maxIterations=iteration_num)
t_cost= model.computeCost(parsedData)
print "INFO: cost for training set =", str(t_cost)
clusterCenters=model.clusterCenters
print "INFO: clusterCenters t=", type(clusterCenters) #list
elif flag_model == "gaussian_mixture_model": # didn't work some native lib issue
print "=================== Gaussian_Mixture_Model ============"
model = GaussianMixture.train(parsedData, k, maxIterations=iteration_num)
print "INFO: model.weights =", model.weights
else:
print "INFO: Training model selection error: no valid ML model selected!"
return
### Save model
save_dir = config.get('app', 'HADOOP_MASTER')+config.get('app', 'HDFS_MODEL_DIR')+'/'+row_id_str
try:
hdfs.ls(save_dir)
#print "find hdfs folder"
hdfs.rmr(save_dir)
#print "all files removed"
except IOError as e:
print "ERROR: I/O error({0}): {1}".format(e.errno, e.strerror)
except:
print "ERROR: Unexpected error:", sys.exc_info()[0]
print "INFO: model saved at hdfs=",save_dir
print "INFO: model type=",type(model)," model=",model
model.save(sc, save_dir)
###load model if needed
#sameModel = SVMModel.load(sc, save_dir)
###
# (true label, keams label, features list, hash)
all_data=samples_rdd.map(lambda t: ( t[0].label, model.predict(t[0].features), t[0].features, t[1] ) ).collect()
true_label_arr = np.asarray([int(x) for x,_,_,_ in all_data])
labels_kmeans = np.asarray([int(x) for _,x,_,_ in all_data])
hash_list = np.asarray([x for _,_,_,x in all_data])
print "INFO: all_data len=",len(all_data),"all_data t=",type(labels_list_all)
print "INFO: true_label_arr.shape=",true_label_arr.shape,"labels_kmeans.shape=",labels_kmeans.shape
print "INFO: true_label_arr t=",type(true_label_arr),"labels_kmeans t=",type(labels_kmeans)
mtx_center=np.asarray(clusterCenters)
features_array_reduced=np.asarray([x.toArray() for _,_,x,_ in all_data])
print "INFO: mtx_center t=",type(mtx_center),"mtx_center.shape=",mtx_center.shape
print "INFO: features_array_reduced t=",type(features_array_reduced),"features_array_reduced.shape",features_array_reduced.shape
#Adjusted Mutual Information between two clusterings
amis=adjusted_mutual_info_score(labels_list_all,labels_kmeans)
print "INFO: Adjusted_mutual_info_score=", amis
#Similarity measure between two clusterings
ars=adjusted_rand_score(labels_list_all,labels_kmeans)
print "INFO: Adjusted_rand_score=", ars
accuracy=0.0
t1 = time()
print 'INFO: training run time: %f' %(t1-t0)
t0 = t1
###############################################
########## plot histogram ######
###############################################
n_clusters=k
plot_col_num = int(math.ceil(math.sqrt(n_clusters)))
figsize = (4*plot_col_num, 3*int(math.ceil(n_clusters*1.0/plot_col_num)))
print "INFO: n_clusters=",n_clusters,",label_dic=",label_dic
print "INFO: plot_col_num=",plot_col_num,",figsize=",figsize,",local_out_dir=",local_out_dir
# kmeans histogram
_, p_true = ml_plot_kmeans_histogram_subfigures(true_label_arr, labels_kmeans, n_clusters, names = label_dic
, plot_col_num = plot_col_num, figsize=figsize, folder = local_out_dir, rid=row_id_str)
# normalized kmeans histogram
_, p_true_norm = ml_plot_kmeans_histogram_subfigures(true_label_arr, labels_kmeans, n_clusters, names = label_dic
, plot_col_num = plot_col_num, figsize=figsize, normalize = True, folder = local_out_dir, rid=row_id_str)
####plot "reverse" histogram with labels ####
num_bars = max(true_label_arr) + 1
figsize = (4*plot_col_num, 3*int(math.ceil(num_bars*1.0/plot_col_num)))
_, p_cluster = ml_plot_kmeans_histogram_subfigures(labels_kmeans, true_label_arr, num_bars, names = label_dic
, plot_col_num = plot_col_num, figsize=figsize, reverse = True, folder = local_out_dir, rid=row_id_str)
#### plot dot figures ####
# dot plot for Kmeans ===========
filename=os.path.join(local_out_dir ,row_id_str+'_cluster.png')
filename_3d=os.path.join(local_out_dir ,row_id_str+'_cluster_3d.json')
ml_plot_kmeans_dot_graph_save_file(features_array_reduced, labels_kmeans, mtx_center, n_clusters, figsize=(10,7), filename=filename
, title='KMeans', filename_3d=filename_3d)
# dot plot for True Labels ===========
filename=os.path.join(local_out_dir ,row_id_str+'_cluster_tl.png')
filename_3d=os.path.join(local_out_dir ,row_id_str+'_cluster_3d_tl.json')
ml_plot_kmeans_dot_graph_save_file(features_array_reduced, true_label_arr, mtx_center, n_clusters, figsize=(10,7), filename=filename
, title='True Labels', filename_3d=filename_3d)
dataset_info={"training_fraction":1, "class_count":n_clusters,"dataset_count":total_sample_count}
# only update db for web request
if fromweb=="1":
#print "database update"
str_sql="UPDATE atdml_document set "+"accuracy = '" \
+"', status = 'learned', processed_date ='"+str(datetime.datetime.now()) \
+"', total_feature_numb='"+str(feature_count) \
+"', perf_measures='{}" \
+"', dataset_info='"+json.dumps(dataset_info) \
+"' where id="+row_id_str
ret=exec_sqlite.exec_sql(str_sql)
print "INFO: Data update done! ret=", str(ret)
else:
print "INFO: accuracy = '"+str(accuracy*100)+"%"
print 'INFO: Finished!'
return 0
def train(row_id_str,
ds_id,
hdfs_feat_dir,
local_out_dir,
ml_opts_jstr,
excluded_feat_cslist,
sp_master,
spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory,
sp_core_max,
zipout_dir,
zipcode_dir,
zip_file_name,
mongo_tuples,
labelnameflag,
fromweb,
training_fraction,
jobname,
model_data_folder,
random_seed=None):
# create zip files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# ML model filename ====
model_fname = os.path.join(model_data_folder, row_id_str + '.pkl')
print "INFO: model_data_folder=", model_data_folder
# create out folders and clean up old model files ====
ml_util.ml_prepare_output_dirs(row_id_str, local_out_dir,
model_data_folder, model_fname)
# init Spark context ====
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
# start here =================================================================== ===============
t0 = time()
# check if ml_opts.has_excluded_feat ==1 ===================================
has_excluded_feat = 0
if not ml_opts_jstr is None:
ml_opts = json.loads(ml_opts_jstr)
if "has_excluded_feat" in ml_opts:
has_excluded_feat = ml_opts["has_excluded_feat"]
# get excluded feature list from mongo ========== ===
if str(has_excluded_feat) == "1" and excluded_feat_cslist is None:
excluded_feat_cslist = ml_util.ml_get_excluded_feat(
row_id_str, mongo_tuples)
print "INFO: excluded_feat_cslist=", excluded_feat_cslist
# source libsvm filename
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
# load feature count file
feat_count_file = libsvm_data_file + "_feat_count"
feature_count = zip_feature_util.get_feature_count(sc, feat_count_file)
print "INFO: feature_count=", feature_count
# load sample RDD from text file
# also exclude selected features in sample ================ =====
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(
sc, libsvm_data_file, feature_count, excluded_feat_cslist)
# collect all data to local for processing ===============
all_data = samples_rdd.collect()
sample_count = len(all_data)
if not random_seed is None and int(random_seed) > 0:
np.random.seed(int(random_seed))
all_data = sorted(all_data, key=lambda x: x[1])
# 2-D array
features_list = [x.features.toArray() for x, _ in all_data]
# label array
labels_list_all = [x.label for x, _ in all_data]
# hash array
hash_list_all = [x for _, x in all_data]
# convert to np array
labels_list_all = array(labels_list_all)
features_array = np.array(features_list)
hash_list_all = np.array(hash_list_all)
#print "features_list=",features_list
# generate sparse matrix (csr) for all samples
features_sparse_mtx = csr_matrix(features_array)
# if ensamble is on, do special split here
### randomly split the samples into training and testing data ===============
X_train_sparse, X_test_sparse, labels_train, labels_test, train_hash_list, test_hash_list = \
cross_validation.train_test_split(features_sparse_mtx, labels_list_all, hash_list_all, test_size=(1-training_fraction) )
# X_test_sparse is scipy.sparse.csr.csr_matrix
testing_sample_count = len(labels_test)
training_sample_count = len(labels_train)
training_lbl_cnt_list = Counter(labels_train)
testing_lbl_cnt_list = Counter(labels_test)
print "INFO: training sample count=", training_sample_count, ", testing sample count=", testing_sample_count, ",sample_count=", sample_count
print "INFO: training label list=", training_lbl_cnt_list, ", testing label list=", testing_lbl_cnt_list
print "INFO: train_hash_list count=", len(
train_hash_list), ", test_hash_list count=", len(test_hash_list)
# random_seed testing
if not random_seed is None:
cnt = 0
for i in train_hash_list:
print i
cnt = cnt + 1
if cnt > 3:
break
#print "INFO: labels_list_all=",labels_list_all # too big
t1 = time()
print 'INFO: running time: %f' % (t1 - t0)
###############################################
###########build learning model ==================================================== ===============
###############################################
### parse parameters and generate the model ###
(clf, model_name) = parse_param_and_get_model(ml_opts)
if model_name == "none":
print "ERROR: model name not found!"
return -1
#####fit the model to training dataset ===============
try:
clf.fit(X_train_sparse, labels_train)
except:
print "ERROR: clf.fit(): clf=", clf
print "ERROR: sys.exc_info:", sys.exc_info()[0]
return -1
print "INFO: model type=", type(clf), " clf=", clf
#### save clf for future use ================== ===============
joblib.dump(clf, model_fname)
# get data from model ================================
coef = None
intercept = None
# get column size =====
try:
if type(clf) in (classes.SVC, classes.NuSVC): # svm didn't have coef_
col_num = clf.support_vectors_.shape[1]
else: #linear only
# coef_ is only available when using a linear kernel
col_num = len(clf.coef_[0])
coef = clf.coef_[0]
intercept = clf.intercept_[0] # only get 1st item?
#print "**model:clf.coef_[0] =",clf.coef_[0]
# save coef_ to Mongo
except Exception as e:
print "Warning: Can't get clf.coef_[0]. e=", e, ", get total features from meta-data"
col_num = 0 #how to get feature number for sparse array?
print "INFO: total feature # in the model: ", col_num
jfeat_coef_dict = {}
# create feature coefficient file ================================
if coef is None:
print "WARNING: model weights not found!"
else:
feat_filename = os.path.join(local_out_dir,
row_id_str + "_feat_coef.json")
print "INFO: feat_filename=", feat_filename
# save coef_arr to mongo ===
#jfeat_coef_dict=save_coef2db(row_id_str, mongo_tuples, coef, intercept, feat_filename, ds_id)
jfeat_coef_dict = ml_util.ml_save_coef_build_feat_coef(
row_id_str, mongo_tuples, coef, intercept, feat_filename, ds_id)
#print "INFO: jfeat_coef_dict=", jfeat_coef_dict
print "INFO: jfeat_coef_dict len=", len(jfeat_coef_dict)
### Evaluating the model on testing dataset ===============
labels_pred = clf.predict(X_test_sparse)
accuracy = clf.score(X_test_sparse, labels_test)
print "INFO: Accuracy = ", accuracy
# filename for false prediction samples ===============
false_pred_fname = os.path.join(local_out_dir,
row_id_str + "_false_pred.json")
print "INFO: false_pred_fname=", false_pred_fname
# build files for false pred & score graph
(score_arr_0, score_arr_1, max_score,
min_score) = ml_build_false_pred(X_test_sparse,
coef,
intercept,
labels_test,
labels_pred,
test_hash_list,
model_name,
jfeat_coef_dict,
false_pred_fname,
row_id_str=row_id_str,
ds_id=ds_id,
mongo_tuples=mongo_tuples)
# save pred output
pred_out_arr = []
for i in range(0, len(labels_test)):
pred_out_arr.append(
(labels_test[i], labels_pred[i], test_hash_list[i]))
pred_ofname = os.path.join(local_out_dir, row_id_str + "_pred_output.pkl")
print "INFO: pred_ofname=", pred_ofname
ml_util.ml_pickle_save(pred_out_arr, pred_ofname)
###################################################
### generate label names (family names) ==================================================== ===============
###################################################
if labelnameflag == 1:
label_dic = ml_util.ml_get_label_dict(row_id_str, mongo_tuples, ds_id)
print "INFO: label_dic =", label_dic
else:
label_dic = {}
label_set = set(labels_list_all)
for label_value in label_set:
label_dic[int(label_value)] = str(int(label_value))
print "INFO: label_dic=", label_dic
labels_list = []
for key in sorted(label_dic):
labels_list.append(label_dic[key])
###############################################
###########plot prediction result figures ==================================================== ===============
###############################################
pred_fname = os.path.join(local_out_dir, row_id_str + "_1" + ".png")
true_fname = os.path.join(local_out_dir, row_id_str + "_2" + ".png")
pred_xlabel = 'Prediction (Single Run)'
true_xlabel = 'True Labels (Single Run)'
test_cnt_dic = ml_util.ml_plot_predict_figures(labels_pred.tolist(),
labels_test.tolist(),
labels_list, label_dic,
testing_sample_count,
pred_xlabel, pred_fname,
true_xlabel, true_fname)
print "INFO: figure files: ", pred_fname, true_fname
#print "INFO: Number of samples in each label is=", test_cnt_dic
roc_auc = None
#fscore=None
perf_measures = None
class_count = len(labels_list)
dataset_info = {
"training_fraction": training_fraction,
"class_count": class_count,
"dataset_count": sample_count
}
#############################################################
###################for 2 class only (plot ROC curve) ==================================================== ===============
#############################################################
if class_count == 2:
# build data file for score graph
score_graph_fname = os.path.join(local_out_dir,
row_id_str + "_score_graph.json")
print "INFO: score_graph_fname=", score_graph_fname
ml_build_pred_score_graph(score_arr_0, score_arr_1, model_name,
score_graph_fname, max_score, min_score)
do_ROC = True
# clean is 0; dirty is 1
reverse_label_dic = dict((v, k) for k, v in label_dic.items())
if 'clean' in reverse_label_dic:
flag_clean = reverse_label_dic['clean']
elif 'benign' in reverse_label_dic:
flag_clean = reverse_label_dic['benign']
elif '0' in reverse_label_dic:
flag_clean = 0
else:
print "WARNING: No ROC curve generated: 'clean' or '0' must be a label for indicating negative class!"
do_ROC = False
if do_ROC:
# calculate fscore ==========
perf_measures = ml_util.calculate_fscore(labels_test, labels_pred)
#fscore=perf_measures["fscore"]
#acc=perf_measures["accuracy"]
#phi=perf_measures["phi"]
print "INFO: perf_measures=", perf_measures
confidence_score = clf.decision_function(X_test_sparse)
#print "INFO:confidence_score=",confidence_score
if flag_clean == 0:
scores = [x for x in confidence_score]
s_labels = [x for x in labels_test]
testing_N = test_cnt_dic[0]
testing_P = test_cnt_dic[1]
else:
scores = [-x for x in confidence_score]
s_labels = [1 - x for x in labels_test]
testing_N = test_cnt_dic[1]
testing_P = test_cnt_dic[0]
# create ROC data file ======== ====
roc_auc = ml_create_roc_files(row_id_str, scores, s_labels,
testing_N, testing_P, local_out_dir,
row_id_str)
perf_measures["roc_auc"] = roc_auc
# only update db for web request ==================================================== ===============
if fromweb == "1":
#print "database update"
str_sql="UPDATE atdml_document set "+"accuracy = '"+str(accuracy*100)+"%" \
+"', status = 'learned', processed_date ='"+str(datetime.datetime.now()) \
+"', perf_measures='"+json.dumps(perf_measures) \
+"', dataset_info='"+json.dumps(dataset_info) \
+"' where id="+row_id_str
ret = exec_sqlite.exec_sql(str_sql)
print "INFO: Sqlite update done! ret=", str(ret)
else:
print "INFO: accuracy = '" + str(accuracy * 100) + "%"
t1 = time()
print 'INFO: running time: %f' % (t1 - t0)
print 'INFO: Train Finished!'
return 0
def save_inherited_ds(row_id_str,
hdfs_src_dir,
hdfs_src_fname,
hdfs_out_dir,
hdfs_out_fname,
sp_master=config.get('spark', 'spark_master'),
spark_rdd_compress=config.get('spark',
'spark_rdd_compress'),
spark_driver_maxResultSize=config.get(
'spark', 'spark_driver_maxResultSize'),
sp_exe_memory=config.get('spark',
'spark_executor_memory'),
sp_core_max=config.get('spark', 'spark_cores_max'),
jobname='extract_dataset:',
join_field_name='md5',
zfname=None,
tblname='',
sql_script='',
hash_list=None):
if len(zfname) > 0:
print "INFO: zip filename=", os.path.basename(zfname)
hash_list = get_hash_list(zfname)
if len(hash_list) <= 0:
print "ERROR: hash list is required!"
return -1
if not row_id_str in jobname:
jobname = jobname + row_id_str
#hash_list=[ ('2','a'),('3','b'),('99','xx'),('101','yy') ]
print 'INFO: hash_list len=', len(hash_list), ", type=", type(hash_list)
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname)
t0 = time()
sqlCtx = SQLContext(sc)
# convert input list into a DF
# TBD hardcode "_label_"
sch = StructType([
StructField(join_field_name, StringType(), True),
StructField("_label_", StringType(), True)
])
plist = sc.parallelize(hash_list)
list_df = sqlCtx.createDataFrame(plist, sch)
# debug only
#list_df.show()
# source filename
src_fname = os.path.join(hdfs_src_dir, hdfs_src_fname)
print "INFO: src fname for source dataset=", os.path.basename(
src_fname), ",join_field_name=", join_field_name
sqlCtx = SQLContext(sc)
src_df = sqlCtx.read.parquet(src_fname)
#src_df.registerTempTable('pTbl')
# get intersect DF
src_df_rt = src_df.join(
list_df, src_df[join_field_name] == list_df[join_field_name],
'right_outer').cache()
# existing data for reuse:
existing_df = src_df_rt.select(src_df['*']).where(
src_df[join_field_name].isNotNull()).cache()
count_df = existing_df.count()
print "INFO: existing_df count=", count_df
''' debug only
print "INFO: src_df="
src_df.show()
print "INFO: existing_df="
existing_df.show()
print "INFO: src_df_rt="
src_df_rt.show()
'''
# generate out folder, clean up if needed
try:
hdfs.mkdir(hdfs_out_dir)
except:
e = sys.exc_info()[0]
print "WARNING: ", e
# output hdfs file name
out_fname = os.path.join(hdfs_out_dir, hdfs_out_fname)
print "INFO: save data from inherited dataset to file=", os.path.basename(
out_fname)
# clean up existing hdfs file
try:
hdfs.rmr(out_fname)
except:
e = sys.exc_info()[0]
print "WARNING: ", e
print "INFO: tbl=", tblname, ",sql=", sql_script
# convert to sql table
existing_df.registerTempTable(tblname)
df2 = sqlCtx.sql(sql_script)
# save as parquet
df2.write.parquet(out_fname)
#df2.printSchema()
new_list = []
# new list which not in src DF for new retrival
row_list = src_df_rt.select(list_df['*'], src_df[join_field_name]).where(
src_df[join_field_name].isNull()).collect()
new_list = [(i[0], i[1]) for i in row_list]
#print "row_list=",row_list
#print "INFO: new_list len=",len(new_list)
''' debug only
test4=sqlCtx.read.parquet(out_fname)
print "reload parquet="
test4.show()
'''
t1 = time()
print 'INFO: running time: %f' % (t1 - t0)
print 'INFO: save_inherited_ds finished!'
return new_list
def train(row_id_str, ds_id, hdfs_feat_dir, local_out_dir, ml_opts_jstr, excluded_feat_cslist
, sp_master, spark_rdd_compress, spark_driver_maxResultSize, sp_exe_memory, sp_core_max
, zipout_dir, zipcode_dir, zip_file_name
, mongo_tuples, labelnameflag, fromweb
, training_fraction, jobname, model_data_folder ):
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir, zipcode_dir, zip_file_name, prefix='zip_feature_util')
print "INFO: zip_file_path=",zip_file_path
# ML model filename ====
model_fname=os.path.join(model_data_folder, row_id_str+'.pkl')
print "INFO: model_data_folder=",model_data_folder
# create out folders and clean up old model files ====
ml_util.ml_prepare_output_dirs(row_id_str,local_out_dir,model_data_folder,model_fname)
# init Spark context ====
sc=ml_util.ml_get_spark_context(sp_master
, spark_rdd_compress
, spark_driver_maxResultSize
, sp_exe_memory
, sp_core_max
, jobname
, [zip_file_path])
t0 = time()
t00 = t0
# check if ml_opts.has_excluded_feat ==1 ===================================
has_excluded_feat=0
if not ml_opts_jstr is None:
ml_opts=json.loads(ml_opts_jstr)
if "has_excluded_feat" in ml_opts:
has_excluded_feat=ml_opts["has_excluded_feat"]
# get excluded feature list from mongo ========== ===
if str(has_excluded_feat) == "1" and excluded_feat_cslist is None:
excluded_feat_cslist=ml_util.ml_get_excluded_feat(row_id_str, mongo_tuples)
print "INFO: excluded_feat_cslist=",excluded_feat_cslist
# source libsvm filename
libsvm_data_file = os.path.join(hdfs_feat_dir , "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
# load feature count file
feat_count_file=libsvm_data_file+"_feat_count"
feature_count=zip_feature_util.get_feature_count(sc,feat_count_file)
print "INFO: feature_count=",feature_count
# load sample RDD from text file
# also exclude selected features in sample ================ =====
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
samples_rdd,feature_count = zip_feature_util.get_sample_rdd(sc, libsvm_data_file, feature_count, excluded_feat_cslist)
all_data = samples_rdd.collect()
sample_count=len(all_data)
# 2-D array
features_list = [x.features.toArray() for x,_ in all_data]
# label array
labels_list_all = [x.label for x,_ in all_data]
# hash array
hash_list_all = [x for _,x in all_data]
# convert to np array
labels_list_all = array(labels_list_all)
features_array = np.array(features_list)
hash_list_all=np.array(hash_list_all)
# generate sparse matrix (csr) for all samples
features_sparse_mtx = csr_matrix(features_array)
### randomly split the samples into training and testing data ===============
X_train_sparse, X_test_sparse, labels_train, labels_test, train_hash_list, test_hash_list = \
cross_validation.train_test_split(features_sparse_mtx, labels_list_all, hash_list_all, test_size=(1-training_fraction) )
# X_test_sparse is scipy.sparse.csr.csr_matrix
testing_sample_count = len(labels_test)
training_sample_count=len(labels_train)
training_lbl_cnt_list=Counter(labels_train)
testing_lbl_cnt_list=Counter(labels_test)
print "INFO: training sample count=",training_sample_count,", testing sample count=",testing_sample_count,",sample_count=",sample_count
print "INFO: training label list=",training_lbl_cnt_list,", testing label list=",testing_lbl_cnt_list
print "INFO: train_hash_list count=",len(train_hash_list),", test_hash_list count=",len(test_hash_list)
t1 = time()
print 'INFO: running time: %f' %(t1-t0)
###############################################
###########build learning model################
###############################################
### parse parameters and generate the model ###
(clf, model_name, api, cv, param_dic) = parse_param_and_get_model(ml_opts)
if model_name == "none":
print "ERROR: model name not found!"
return -1
#param_jobj=json.loads(ml_opts_jstr);
#print "param_jobj=",param_jobj
########################################################
##########Grid Search with cross validation#############
########################################################
json2save={}
json2save["rid"]=int(row_id_str)
json2save["key"]="cv_result"
#json2save["param_str"]=ml_opts_jstr
json2save["param_dic"]=param_dic
cv_grid=[]
if api == "centralized":
#########run with Scikit-learn API (for comparison)######
print "INFO: ******************Grid Search with Scikit-learn API************"
t0 = time()
# Set the parameters by cross-validation
#tuned_parameters = [{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000, 100000]}]
#tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4], \
# 'C': [1, 10, 100, 1000]}, \
# {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]
scores = ['accuracy']
json2save["scores"]=scores
#print json2save
for score in scores: # for one item only? score=accuracy
print("INFO: # Tuning hyper-parameters for %s" % score)
#print()
grid = grid_search.GridSearchCV(estimator = clf, param_grid = param_dic, cv=cv, scoring= score)
grid.fit(X_train_sparse, labels_train)
print "INFO: Best parameters set found on development set:"
print "INFO: grid.best_params_=",grid.best_params_
print "INFO: Grid scores on development set:"
for key in grid.best_params_:
print "INFO: best_params["+key+"]=", grid.best_params_[key]
if key.lower()=="regtype":
ml_opts['regularization']=str(grid.best_params_[key]) # add best param to
else:
ml_opts[key.lower()]=str(grid.best_params_[key]) # add best param to
# save best param to db as json string
j_str=json.dumps(ml_opts);
json2save["param_str"]=j_str;
print "INFO: grid_scores_ with params:"
for params, mean_score, scores in grid.grid_scores_:
print "INFO: %0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params)
#outstr='%s,%0.3f,%0.03f,%s' % (params,mean_score, scores.std() * 2,"Selected" if params==grid.best_params_ else "")
outj={}
outj["param"]=params
outj["average_accuracy"]="%0.3f" % (mean_score)
outj["std_deviation"]="%0.3f" % (scores.std() * 2)
outj["selected"]="%s" % ("Selected" if params==grid.best_params_ else "")
cv_grid.append(outj)
clf_best = grid.best_estimator_
t1 = time()
############# END run with SKlearn ######
print 'INFO: Grid Search with SKlearn running time: %f' %(t1-t0)
t0 = time()
else:
#############run with SPARK######
print "INFO: ******************Grid Search with SPARK************"
all_comb_list_of_dic = get_all_combination_list_of_dic(param_dic)
print "INFO: Total number of searching combinations=", len(all_comb_list_of_dic)
#print "all_comb_list_of_dic: ", all_comb_list_of_dic
params_rdd = sc.parallelize(all_comb_list_of_dic)
###broad cast clf, traning data, testing data to all workers###
X_broadcast = sc.broadcast(X_train_sparse)
y_broadcast = sc.broadcast(labels_train)
clf_broadcast = sc.broadcast(clf)
### Grid Search with CV in multiple workers ###
models = params_rdd.map(lambda x: learn_with_params(clf_broadcast.value, X_broadcast.value, y_broadcast.value, cv, x)).sortByKey(ascending = False).cache()
(ave_accuracy, (clf_best, p_dic_best, std2)) = models.first()
# output results #
print "INFO: Best parameters set found for ", model_name, " is: "
print "INFO: ",
for key in p_dic_best:
print key, " = ", p_dic_best[key],
if key.lower()=="regtype":
ml_opts['regularization']=str(p_dic_best[key])
else:
ml_opts[key.lower()]=str(p_dic_best[key]) # add best param to
# save best param to db as json string
print ""
j_str=json.dumps(ml_opts);
json2save["param_str"]=j_str;
print "INFO: Average accuracy with CV = ", cv, ": ", ave_accuracy
######## print complete report #######
print "INFO: Grid scores on development set:"
all_results = models.collect()
for i in range(0, len(all_results)):
(ave_accu_i, (clf_i, p_dic_i, std2_i)) = all_results[i]
print "INFO: ",ave_accu_i, " for ", p_dic_i
print "INFO: %0.3f (+/-%0.03f) for " % (ave_accu_i, std2_i), p_dic_i
#outstr='%s,%0.3f,%0.03f,%s' % ( p_dic_i, ave_accu_i, std2_i, "Selected" if p_dic_i==p_dic_best else "")
outj={}
outj["param"]=p_dic_i
outj["average_accuracy"]="%0.3f" % (ave_accu_i)
outj["std_deviation"]="%0.3f" % (std2_i)
outj["selected"]="%s" % ("Selected" if p_dic_i==p_dic_best else "")
cv_grid.append(outj)
print " "
t1 = time()
############# END run with SPARK######
print 'INFO: Grid search with SPARK running time: %f' %(t1-t0)
##################################################################################
#print "cv_grid=",cv_grid
#json2save["cv_grid_title"]='param,average_accuracy,std_deviation,selected'
json2save["cv_grid_data"]=cv_grid
json2save['clf_best']=str(clf_best).replace("\n","").replace(" ","")
cv_result=json.dumps(json2save)
#print "INFO: cv_result=",cv_result
filter='{"rid":'+row_id_str+',"key":"cv_result"}'
upsert_flag=True
## write to mongoDB.myml.dataset_info, ignore doc with duplicated key
# db.dataset_info.createIndex({"rid":1,"key":1},{unique:true})
ret=query_mongo.upsert_doc_t(mongo_tuples,filter,cv_result,upsert_flag)
print "INFO: Upsert count for cv_result: ret=",ret
##################################################################################
##########Retrain with best model for training set and output results#############
##################################################################################
print "INFO: **********Retrain with best model for training set and output results************"
clf_best.fit(X_train_sparse, labels_train)
#### save clf_best for future use ####
#joblib.dump(clf_best, model_data_folder + row_id_str+'.pkl')
joblib.dump(clf_best, model_fname)
### Evaluating the model on testing data
labels_pred = clf_best.predict(X_test_sparse)
accuracy = clf_best.score(X_test_sparse, labels_test)
print "INFO: Accuracy = ", accuracy
######################################the rest of the code is the same as train_sklean.py (replace clf with clf_best)#####################################################################
clf=clf_best
print "INFO: model type=",type(clf)," clf=",clf
# get data from model ================================
coef=None
intercept=None
try:
if type(clf) in ( classes.SVC , classes.NuSVC) :# svm didn't have coef_
col_num=clf.support_vectors_.shape[1]
else: #linear only
# coef_ is only available when using a linear kernel
col_num = len(clf.coef_[0])
coef=clf.coef_[0]
intercept=clf.intercept_[0] # only get 1st item?
#print "**model:clf.coef_[0] =",clf.coef_[0]
except Exception as e:
print "WARNING: Can't get clf.coef_[0]. e=",e,", get total features from meta-data"
col_num = 0 #how to get feature number for sparse array?
print "INFO: total feature # in the model: ", col_num
jfeat_coef_dict={}
# create feature coefficient file ================================
if coef is None:
print "WARNING: model weights not found!"
else:
feat_filename=os.path.join(local_out_dir,row_id_str+"_feat_coef.json")
print "INFO: feat_filename=",feat_filename
# save coef_arr to mongo & create jfeat_coef_dict===
jfeat_coef_dict=ml_util.ml_save_coef_build_feat_coef(row_id_str, mongo_tuples, coef, intercept, feat_filename, ds_id)
#print "INFO: jfeat_coef_dict=", jfeat_coef_dict
print "INFO: jfeat_coef_dict len=", len(jfeat_coef_dict )
# filename for false pred
false_pred_fname=os.path.join(local_out_dir,row_id_str+"_false_pred.json")
print "INFO: false_pred_fname=", false_pred_fname
# build files for false pred & score graph
(score_arr_0, score_arr_1, max_score,min_score)=ml_build_false_pred(X_test_sparse,coef,intercept
, labels_test, labels_pred, test_hash_list, model_name, jfeat_coef_dict, false_pred_fname)
# save pred output
pred_out_arr=[]
for i in range(0,len(labels_test)):
pred_out_arr.append((labels_test[i], labels_pred[i], test_hash_list[i]))
pred_ofname=os.path.join(local_out_dir,row_id_str+"_pred_output.pkl")
print "INFO: pred_ofname=", pred_ofname
ml_util.ml_pickle_save(pred_out_arr,pred_ofname)
###################################################
### generate label names (family names) ###########
### connect to database to get the column list which contains all column number of the corresponding feature####
###################################################
if labelnameflag == 1:
key = "dic_name_label"
jstr_filter='{"rid":'+row_id_str+',"key":"'+key+'"}'
jstr_proj='{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter='{"rid":'+ds_id+',"key":"'+key+'"}'
doc=query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_list = doc['value']
label_dic = {}
for i in range(0, len(dic_list)):
for key in dic_list[i]:
label_dic[dic_list[i][key]] = key.encode('UTF8')
print "INFO: label_dic:", label_dic
else:
label_dic = {}
label_set = set(labels_list_all)
for label_value in label_set:
label_dic[int(label_value)] = str(int(label_value))
print "INFO: ******generated label_dic:", label_dic
labels_list = []
for key in sorted(label_dic):
labels_list.append(label_dic[key])
### generate sample numbers of each family in testing data###
testing_sample_number = len(labels_test)
print "INFO: testing_sample_number=", testing_sample_number
test_cnt_dic = {}
for key in label_dic:
test_cnt_dic[key] = 0
for i in range (0, testing_sample_number):
for key in label_dic:
if labels_test[i] == key:
test_cnt_dic[key] = test_cnt_dic[key] + 1
print "INFO: Number of samples in each label is=", test_cnt_dic
###############################################
###########plot prediction result figure#######
###############################################
pred_fname=os.path.join(local_out_dir,row_id_str+"_1"+".png")
true_fname=os.path.join(local_out_dir,row_id_str+"_2"+".png")
pred_xlabel='Prediction (Single Run)'
true_xlabel='True Labels (Single Run)'
test_cnt_dic=ml_util.ml_plot_predict_figures(labels_pred.tolist(), labels_test.tolist(), labels_list, label_dic, testing_sample_count
, pred_xlabel, pred_fname, true_xlabel, true_fname)
print "INFO: figure files: ", pred_fname, true_fname
print "INFO: Number of samples in each label is=", test_cnt_dic
roc_auc=None
#fscore=None
perf_measures=None
class_count=len(labels_list)
dataset_info={"training_fraction":training_fraction, "class_count":class_count,"dataset_count":sample_count}
#############################################################
###################for 2 class only (plot ROC curve)#########
#############################################################
if len(labels_list) == 2:
# build data file for score graph
score_graph_fname=os.path.join(local_out_dir,row_id_str+"_score_graph.json")
print "INFO: score_graph_fname=", score_graph_fname
ml_build_pred_score_graph(score_arr_0,score_arr_1,model_name, score_graph_fname,max_score,min_score)
do_ROC=True
reverse_label_dic = dict((v,k) for k, v in label_dic.items())
if 'clean' in reverse_label_dic:
flag_clean = reverse_label_dic['clean']
elif 'benign' in reverse_label_dic:
flag_clean = reverse_label_dic['benign']
elif '0' in reverse_label_dic:
flag_clean = 0
else:
print "No ROC curve generated: 'clean' or '0' must be a label for indicating negative class!"
do_ROC=False
if do_ROC:
# calculate fscore ==========
perf_measures=ml_util.calculate_fscore(labels_test, labels_pred)
print "INFO: perf_measures=",perf_measures
confidence_score = clf_best.decision_function(X_test_sparse)
if flag_clean == 0:
scores = [x for x in confidence_score]
s_labels = [x for x in labels_test]
testing_N = test_cnt_dic[0]
testing_P = test_cnt_dic[1]
else:
scores = [-x for x in confidence_score]
s_labels = [1-x for x in labels_test]
testing_N = test_cnt_dic[1]
testing_P = test_cnt_dic[0]
# create ROC data file ======== ====
roc_auc=ml_create_roc_files(row_id_str, scores, s_labels, testing_N, testing_P
, local_out_dir, row_id_str)
perf_measures["roc_auc"]=roc_auc
# only update db for web request
if fromweb=="1":
#print "database update"
str_sql="UPDATE atdml_document set "+"accuracy = '"+str(accuracy*100)+"%" \
+"', status = 'learned', processed_date ='"+str(datetime.datetime.now()) \
+"',ml_opts='"+j_str \
+"', perf_measures='"+json.dumps(perf_measures) \
+"', dataset_info='"+json.dumps(dataset_info) \
+"' where id="+row_id_str
ret=exec_sqlite.exec_sql(str_sql)
print "INFO: Data update done! ret=", str(ret)
else:
print "INFO: accuracy = '"+str(accuracy*100)+"%"
print 'INFO: total running time: %f' %(t1-t00)
print 'INFO: Finished!'
return 0
def mrun(row_id_str, ds_id, hdfs_feat_dir, local_out_dir, ml_opts_jstr, excluded_feat_cslist
, sp_master, spark_rdd_compress, spark_driver_maxResultSize, sp_exe_memory, sp_core_max
, zipout_dir, zipcode_dir, zip_file_name
, mongo_tuples, fromweb
, training_fraction, jobname, run_number, bin_number ):
### generate data folder and out folder, clean up if needed
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir, zipcode_dir, zip_file_name, prefix='zip_feature_util')
print "INFO: zip_file_path=",zip_file_path
# get_spark_context
sc=ml_util.ml_get_spark_context(sp_master
, spark_rdd_compress
, spark_driver_maxResultSize
, sp_exe_memory
, sp_core_max
, jobname
, [zip_file_path])
t0 = time()
# check if ml_opts.has_excluded_feat ==1 ===================================
has_excluded_feat=0
if not ml_opts_jstr is None:
ml_opts=json.loads(ml_opts_jstr)
if "has_excluded_feat" in ml_opts:
has_excluded_feat=ml_opts["has_excluded_feat"]
#print "has_excluded_feat=",has_excluded_feat,",excluded_feat_cslist=",excluded_feat_cslist
# get excluded feature list from mongo ========== ===
if str(has_excluded_feat) == "1" and excluded_feat_cslist is None:
key = "feature_excluded"
jstr_filter='{"rid":'+row_id_str+',"key":"'+key+'"}'
jstr_proj='{"value":1}'
# get from own id (not from parent dataset id)
#print "jstr_filter=",jstr_filter,",jstr_proj=",jstr_proj
doc=query_mongo.find_one(args.ip_address, args.port, args.db_name, args.tb_name, username, password, jstr_filter, jstr_proj)
#print "feature_excluded=",doc
if not doc is None and 'value' in doc:
excluded_feat_cslist = ','.join(str(i) for i in doc['value'])
print "INFO: excluded_feat_cslist=",excluded_feat_cslist
### generate Labeled point
#libsvm_data_file = data_folder + "libsvm_data"
# filename for featured data
libsvm_data_file = os.path.join(hdfs_feat_dir , "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
# load feature count file
feat_count_file=libsvm_data_file+"_feat_count"
feature_count=zip_feature_util.get_feature_count(sc,feat_count_file)
print "INFO: feature_count=",feature_count
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
# load sample RDD from text file
# also exclude selected features in sample ================ =====
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
samples_rdd, feature_count=zip_feature_util.get_sample_rdd(sc, libsvm_data_file, feature_count, excluded_feat_cslist)
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
# get distinct label list
labels_list_all = samples_rdd.map(lambda p: p[0].label).distinct().collect()
#labels_list_all = samples_rdd.map(lambda p: p.label).collect()
t1 = time()
print "INFO: labels_list_all=",labels_list_all
#print "INFO: training and testing samples generated!"
print 'INFO: data generating time: %f' %(t1-t0)
t0 = t1
### generate label names (family names) #####
### connect to database to get the column list which contains all column number of the corresponding feature####
label_set = set(labels_list_all)
class_num = len(label_set)
#class_num = len(labels_list)
if class_num > 2:
print "INFO:Number of classes=", class_num
###############################################
###########build learning model################
###############################################
### get the parameters###
print "INFO: ============Learning Algorithm and Parameters============="
#param_dict = json.loads(ml_opts_jstr)
flag_model = ml_opts['learning_algorithm'] # 1: linear_svm_with_sgd; 2: logistic_regression_with_lbfgs; 3: logistic_regression_with_sgd
C = eval(ml_opts['c'])
iteration_num = ml_opts['iterations']
regularization = ml_opts['regularization']
print "INFO: Learning Algorithm: ", flag_model
print "INFO: C = ", C
print "INFO: iteration = ", iteration_num
print "INFO: regType = ", regularization
t0 = time()
accuracy_array = np.zeros(run_number)
for rnd in range (0, run_number):
### generate training and testing data
training_rdd, testing_rdd = samples_rdd.randomSplit([training_fraction, 1-training_fraction])
training_rdd=training_rdd.map(lambda p:p[0])# keep LabeledPoint only
training_rdd.cache()
testing_rdd.cache()
training_sample_count = training_rdd.count()
regP = C/float(training_sample_count)
print "INFO: Calculated: regParam = ", regP
### build model ###
if flag_model == "linear_svm_with_sgd":
### 1: linearSVM
print "INFO: ====================1: Linear SVM============="
model_classification = SVMWithSGD.train(training_rdd, regParam=regP, iterations=iteration_num, regType=regularization) # regParam = 1/(sample_number*C)
#print model_classification
elif flag_model == "logistic_regression_with_lbfgs":
### 2: LogisticRegressionWithLBFGS
print "INFO: ====================2: LogisticRegressionWithLBFGS============="
model_classification = LogisticRegressionWithLBFGS.train(training_rdd, regParam=regP, iterations=iteration_num, regType=regularization, numClasses=class_num) # regParam = 1/(sample_number*C)
elif flag_model == "logistic_regression_with_sgd":
### 3: LogisticRegressionWithLBFGS
print "INFO: ====================3: LogisticRegressionWithSGD============="
model_classification = LogisticRegressionWithSGD.train(training_rdd, regParam=regP, iterations=iteration_num, regType=regularization) # regParam = 1/(sample_number*C)
else:
print "ERROR: Training model selection error: no valid ML model selected!"
return
### Evaluating the model on testing data
labelsAndPreds = testing_rdd.map(lambda p: (p[0].label, model_classification.predict(p[0].features)))
labelsAndPreds.cache()
testing_sample_number = testing_rdd.count()
testErr = labelsAndPreds.filter(lambda (v, p): v != p).count() / float(testing_sample_number)
accuracy = 1 - testErr
accuracy_array[rnd] = accuracy
print "INFO: current round=", rnd
print "INFO: Accuracy=", accuracy
########################below: same as train_skLean_multi_run.py#####################################
###############################################
#######plot distribution and variance##########
###############################################
plt.figure(1)
num_bins = bin_number ####10 is default
n, bins, patches = plt.hist(accuracy_array, num_bins, normed=1, facecolor='green', alpha=0.5)
ave = np.mean(accuracy_array)
print "INFO: Accuracy mean=", ave
variance = np.std(accuracy_array)
print "INFO: Accuracy variance=", variance
#print "INFO: bins: ", bins
# add a 'best fit' line
y = mlab.normpdf(bins, ave, variance)
#print "INFO: y: ", y
plt.plot(bins, y, 'r--')
plt.title('Accuracy distribution of '+str(run_number)+' runs:')
plt.xlabel('Accuracy Values')
plt.ylabel('Probability / Accuracy bar width')
#plt.savefig(local_out_dir+file_name_given+"_var_"+str(run_number)+".png")
plt.savefig(os.path.join(local_out_dir, row_id_str+"_var_"+str(run_number)+".png"))
# create ROC data for graph ====================
all_json=[]
barp_arr=[] #n
disp_arr=[] #y
last_idx=0
for idx,ht in enumerate(n): # n is bar height
#print "INFO: mrun bar=",idx, bins[idx], bins[idx+1],((bins[idx]+bins[idx+1])/2)
barp_arr.append([ ((bins[idx]+bins[idx+1])/2.0),n[idx]]) # mid point for x axis
if not math.isnan(y[idx]):
disp_arr.append([bins[idx],y[idx]])
last_idx=idx
#print "INFO: ",last_idx+1, bins[last_idx+1], y[last_idx+1]
if not math.isnan(y[last_idx+1]):
disp_arr.append([bins[last_idx+1],y[last_idx+1]])
#print "barp_arr=", barp_arr
#print "disp_arr=", disp_arr
#bar
bar_json={}
bar_json["values"]=barp_arr
bar_json["key"]='Mutil-Run Accuracy' #
bar_json["type"]="bar" # light blue
bar_json["yAxis"]=1
all_json.append(bar_json)
#distribution
if len(disp_arr)>0:
dis_json={}
dis_json["values"]=disp_arr
dis_json["key"]='Normal Distribution' #
dis_json["type"]="line" # light blue
dis_json["yAxis"]=1
all_json.append(dis_json)
mrun_jfile = os.path.join(local_out_dir, row_id_str+"_mrun.json")
#print "INFO: all_json=",all_json
print "INFO: mrun_jfile=",mrun_jfile
if os.path.exists(mrun_jfile):
try:
os.remove(mrun_jfile)
except OSError, e:
print ("Error: %s - %s." % (e.mrun_jfile,e.strerror))
def feat_extr_ngram(row_id_str,
hdfs_dir_list,
hdfs_feat_dir,
model_data_folder,
sp_master,
spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory,
sp_core_max,
zipout_dir,
zipcode_dir,
zip_file_name,
mongo_tuples,
fromweb,
label_arr,
metadata_count,
label_idx,
data_idx,
pattern_str,
ln_delimitor,
data_field_list,
jkey_dict,
jobname,
num_gram,
feature_count_threshold,
token_dict=None,
HDFS_RETR_DIR=None,
remove_duplicated="N",
cust_featuring=None,
cust_featuring_params=None,
local_out_dir=None,
filter_ratio=None,
binary_flag=True):
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_util.ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
user_custom=cust_featuring)
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
# log time ================================================================ ================
t0 = time()
# input filename
input_filename = "*"
ext_type = '.gz'
gz_list = None
convert2dirty = "N"
if not ',' in hdfs_dir_list: # single dir having *.gz ==== =========
# read raw data from HDFS as .gz format ==========
rdd_files = os.path.join(hdfs_dir_list, input_filename + ext_type)
# check if gz files in hdfs ============
try:
gz_list = hdfs.ls(hdfs_dir_list)
print "INFO: check hdfs folder=", hdfs_dir_list
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(e.errno, e.strerror)
except:
print "WARNING: Error at checking HDFS file:", sys.exc_info()[0]
# use whole folder
if gz_list is None or len(gz_list) == 0:
print "ERROR: No file found by ", input_filename + ext_type #,", use",hdfs_dir_list,"instead"
return -2
elif len(gz_list) == 1:
# use dir as filename
rdd_files = hdfs_dir_list[0:-1]
else: # multiple dirs ==== =========
rdd_files = ""
cnt = 0
temp_lbl_list = []
comma = ""
print "INFO: before label_arr=", label_arr
# check each folder
for dr in hdfs_dir_list.split(','):
#print "****=",dr
if not len(dr) > 0:
continue
try:
# remove space etc.
dr = dr.strip()
fdr = os.path.join(HDFS_RETR_DIR, dr)
#print "fdr=",fdr
# ls didn't like "*"
if '*' in fdr:
#gz_list=hdfs.ls(fdr.replace("*",""))
dn = os.path.dirname(fdr).strip()
bn = os.path.basename(fdr).strip()
#print "dn=",dn,",bn=",bn
# get all names under folder and do filtering
gz_list = fnmatch.filter(hdfs.ls(dn), '*' + bn)
#print "gz_list=",gz_list
else:
gz_list = hdfs.ls(fdr)
cnt = cnt + len(gz_list)
if len(gz_list) > 0:
rdd_files = rdd_files + comma + fdr
comma = ","
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(
e.errno, e.strerror)
except:
print "WARNING: Error at checking HDFS file:", sys.exc_info(
)[0]
# use whole folder
if cnt is None or cnt == 0:
print "ERROR: No file found at", rdd_files
return -2
else:
print "INFO: total file count=", cnt
# set convert flag only when multiple dir and label_arr has dirty label
#if label_arr is None: # create label arr if None
# label_arr=temp_lbl_list
if not label_arr is None and len(
label_arr) == 2 and label_arr[1] == "dirty":
convert2dirty = "Y"
print "INFO: rdd_files=", rdd_files
txt_rdd = sc.textFile(rdd_files) #, use_unicode=False
total_input_count = txt_rdd.count()
print "INFO: Total input sample count=", total_input_count
# debug only
#for x in txt_rdd.collect():
# print "t=",x
print "INFO: hdfs_dir_list=", hdfs_dir_list
print "INFO: label_arr=", label_arr
print "INFO: feature_count_threshold=", feature_count_threshold
#jkey_dict={"meta_list":["label","md5","mdate"], "data_key":"logs"}
# this dict depends on the format of input data
if not data_field_list is None:
jkey_dict = json.loads(jkey_dict)
data_key = jkey_dict["data_key"]
meta_list = jkey_dict["meta_list"]
metadata_count = len(meta_list)
data_idx = metadata_count
print "INFO: jkey_dict=", jkey_dict
print "INFO: meta_list=", meta_list
print "INFO: data_key=", data_key
print "INFO: data_field_list=", data_field_list
print "INFO: metadata_count=", metadata_count
featured_rdd = txt_rdd \
.map(lambda x: preprocess_json(x,meta_list,data_key,data_field_list)) \
.filter(lambda x: len(x) > metadata_count) \
.filter(lambda x: type(x[metadata_count]) is list) \
.map(lambda x: feature_extraction_ngram(x, data_idx, MAX_FEATURES, num_gram)) \
.filter(lambda x: len(x) > metadata_count) \
.filter(lambda x: type(x[metadata_count]) is dict) \
.filter(lambda x: type(x[metadata_count+1]) is dict) \
.filter(lambda x: len(x[metadata_count])> int(feature_count_threshold) ) \
.cache()
#print "INFO: featured_rdd="
#for x in featured_rdd.collect():
# print "INFO: **** f=",x
# user custom code for featuring ============================================= ==========
# input txt_rdd format (string): each text row for each sample
# output featured_rdd format (list):[meta-data1,meta-data2,..., hash_cnt_dic, hash_str_dic]
elif not cust_featuring is None and len(cust_featuring) > 0:
user_module = None
user_func = None
user_func_dnn = None
# load user module =======
try:
modules = map(__import__, [CUSTOM_PREFIX + cust_featuring])
user_module = modules[0]
user_func = getattr(user_module, CUSTOM_FUNC)
except Exception as e:
print "ERROR: module=", CUSTOM_PREFIX + cust_featuring
print "ERROR: user module error.", e.__doc__, e.message
return -101
try:
jparams = json.loads(cust_featuring_params)
if jparams and 'n-gram' in jparams:
num_gram = jparams['n-gram']
elif jparams and 'ngram' in jparams:
num_gram = jparams['ngram']
if jparams and 'binary_flag' in jparams:
binary_flag = eval(jparams['binary_flag'])
except Exception as e:
print "ERROR: user params error.", e.__doc__, e.message
return -200
# convert feast into array. output format: [ meta1,meta2,..., [feat1,feat2,...]]
tmp_rdd = txt_rdd.map(lambda x: user_func(x, cust_featuring_params)) \
.filter(lambda x: len(x) > metadata_count) \
.filter(lambda x: type(x[metadata_count]) is list).cache()
print " tmp_rdd cnt=", tmp_rdd.count(
), ",ix=", data_idx, ",max f=", MAX_FEATURES, "ngram=", num_gram
print "take(1) rdd=", tmp_rdd.take(1)
# TBD for multivariant output format: [ meta1,meta2,..., [[feat1,feat2,...],[feat1,feat2,...],...]]
# TBD only for num_gram available
# for traditional ML, feat in a dict
# output format: [ meta1,meta2,..., [[feat1,feat2,...],[feat1,feat2,...],...]]
featured_rdd = tmp_rdd \
.map(lambda x: feature_extraction_ngram(x, data_idx, MAX_FEATURES, num_gram)) \
.filter(lambda x: len(x) > metadata_count) \
.filter(lambda x: type(x[metadata_count]) is dict) \
.filter(lambda x: type(x[metadata_count+1]) is dict) \
.filter(lambda x: len(x[metadata_count])> int(feature_count_threshold) ) \
.cache()
all_hashes_cnt_dic = None
all_hash_str_dic = None
all_hashes_seq_dic = None
else:
print "INFO: pattern_str=", pattern_str + "<--"
print "INFO: ln_delimitor=", ln_delimitor + "<--"
print "INFO: label_idx=", label_idx
print "INFO: data_idx=", data_idx
print "INFO: metadata_count=", metadata_count
print "INFO: filter_ratio=", filter_ratio
# filter top and least percentage of feature
if not filter_ratio is None and filter_ratio > 0 and filter_ratio < 1:
# check total count here before continue
upper_cnt = total_input_count * (1 - filter_ratio)
lower_cnt = total_input_count * filter_ratio
# set limit for lower bound. if total count is large, lower_cnt may exclude all features...
# max lower count = min( MAX_FILTER_LOWER_CNT, total_input_count/100 )
if not MAX_FILTER_LOWER_CNT is None and lower_cnt > MAX_FILTER_LOWER_CNT:
if MAX_FILTER_LOWER_CNT > total_input_count / 100:
lower_cnt = total_input_count / 100
else:
lower_cnt = MAX_FILTER_LOWER_CNT
print "INFO: filtering by count, upper bound=", upper_cnt, ",lower bound=", lower_cnt
# find unique feature, count them, remove them if in highest and lowest % and then create a dict
f_feat_set = Set (txt_rdd.map(lambda x:x.split(ln_delimitor)).flatMap(lambda x:Set(x[metadata_count:])) \
.map(lambda x:(x,1)).reduceByKey(lambda a, b: a + b) \
.filter(lambda x:x[1]<= upper_cnt and x[1]>= lower_cnt) \
.map(lambda x:x[0]).collect() )
print "INFO: f_feat_set len=", len(f_feat_set)
broadcast_f_set = sc.broadcast(f_feat_set)
#txt_rdd=txt_rdd.map(lambda x: filter_by_list(x, metadata_count,ln_delimitor, broadcast_f_list.value ))
txt_rdd=txt_rdd.map(lambda x: x.split(ln_delimitor)) \
.map(lambda x: x[:metadata_count]+ [w for w in x[metadata_count:] if w and w in broadcast_f_set.value]) \
.map(lambda x: ln_delimitor.join(x))
# preprocess by pattern matching and then extract n-gram features #.encode('UTF8')
# input txt_rdd format (string): meta-data1\tmeta-data2\t...\tdataline1\tdataline2\t...datalineN\n
# output featured_rdd format (list):[meta-data1,meta-data2,..., hash_cnt_dic, hash_str_dic]
# hash_cnt_dic: {hash,hash:count,...} hash_str_dic: {hash: 'str1',... }
tmp_rdd = txt_rdd \
.map(lambda x: preprocess_pattern(x, metadata_count, pattern_str, ln_delimitor \
, label_idx, label_arr, convert2dirty )) \
.filter(lambda x: len(x) > metadata_count) \
.filter(lambda x: type(x[metadata_count]) is list) #.cache() memory issue...
#tmp_rdd_count=tmp_rdd.count()
#print "INFO: After preprocessing count=",tmp_rdd_count
featured_rdd = tmp_rdd \
.map(lambda x: feature_extraction_ngram(x, data_idx, MAX_FEATURES, num_gram)) \
.filter(lambda x: len(x) > metadata_count) \
.filter(lambda x: type(x[metadata_count]) is dict) \
.filter(lambda x: type(x[metadata_count+1]) is dict) \
.filter(lambda x: len(x[metadata_count])> int(feature_count_threshold) ) \
.cache()
#feat_rdd_count=featured_rdd.count()
#print "INFO: After featuring count=",feat_rdd_count
all_hashes_cnt_dic = None
all_hash_str_dic = None
all_hashes_seq_dic = None
#get all hashes and total occurring count ===============
# all_hashes_cnt_dic: {'hash,hash': total count,... }
if all_hashes_cnt_dic is None:
#all_hashes_cnt_dic = featured_rdd.map(lambda x: x[metadata_count]).reduce(lambda a, b: combine_dic_cnt(a, b))
all_hashes_cnt_dic = dict(
featured_rdd.flatMap(lambda x: x[metadata_count].items()).
reduceByKey(lambda a, b: a + b).collect())
#get all hashes and their extracted string ===============
# all_hash_str_dic: {hash:'str1', ...
if all_hash_str_dic is None:
#all_hash_str_dic = featured_rdd.map(lambda x: x[metadata_count+1]).reduce(lambda a, b: combine_dic(a, b))
all_hash_str_dic = dict(
featured_rdd.flatMap(
lambda x: x[metadata_count + 1].items()).distinct().collect())
# get all labels into an array =============== provided by parameter?
if label_arr is None:
# will force "clean" be 0 here
label_arr = sorted(
featured_rdd.map(
lambda x: x[label_idx].lower()).distinct().collect())
# debug only
print "INFO: label_arr.=", json.dumps(sorted(label_arr))
# save labels to hdfs as text file==================================== ============
hdfs_folder = hdfs_feat_dir #+ "/" # "/" is needed to create the folder correctly
print "INFO: hdfs_folder=", hdfs_folder
try:
hdfs.mkdir(hdfs_folder)
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(e.errno, e.strerror)
except:
print "WARNING: Unexpected error at mkdir:", sys.exc_info()[0]
# clean up metadata_file
metadata_file = os.path.join(hdfs_folder, metadata) #"metadata"
print "INFO: metadata_file=", metadata_file
try:
hdfs.rmr(metadata_file)
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(e.errno, e.strerror)
except:
print "WARNING: Unexpected error at rmr():", sys.exc_info()[0]
sc.parallelize(label_arr, 1).saveAsTextFile(metadata_file)
#remap all hash values to continuous key/feature number ==============
# all_hashes_seq_dic: { hash : sequential_numb }
if all_hashes_seq_dic is None:
all_hashes_seq_dic = {}
remap2seq(
all_hashes_cnt_dic,
all_hashes_seq_dic) #all_hashes_seq_dic has continuous key number
#print "all_hashes_seq_dic=",all_hashes_seq_dic
total_feature_numb = len(all_hashes_seq_dic)
print "INFO: Total feature count=", len(all_hashes_seq_dic)
# featured_rdd (list): [meta-data1,meta-data2,..., hash_cnt_dic, hash_str_dic]
# seq_featured_rdd(list): [meta-data1,meta-data2,..., hash_cnthsh_dict, hash_str_dic] (feat id in sorted sequence)
# hash_cnt_dic: {hash: count} hash_str_dic: {hash: 'str1,str2...' }
# set binary_flag to True, all feature:value will be 1
broadcast_dic = sc.broadcast(all_hashes_seq_dic)
seq_featured_rdd = featured_rdd.map(lambda x: convert2seq(
x, label_idx, data_idx, broadcast_dic.value, binary_flag=binary_flag)
).cache()
# get hash_cnthsh_dict then flatMap and reduce to (feat id, count)
ct_rdd = seq_featured_rdd.flatMap(lambda x: [(i[0], i[1]) for i in x[
data_idx].iteritems()]).reduceByKey(lambda a, b: a + b)
# sorted by feature id as int
feat_sample_count_arr = ct_rdd.sortBy(lambda x: int(x[0])).map(
lambda x: x[1]).collect()
# sort after collect may fail when rdd is huge
#feat_sample_count_arr=[]
#for i in sorted(ct_rdd.collect(), key=lambda t: int(t[0])):
# feat_sample_count_arr.append(i[1])
print "INFO: feat_sample_count_arr len=", len(feat_sample_count_arr)
# save feat_sample_count_arr data ==================================== ============
filter = '{"rid":' + row_id_str + ',"key":"feat_sample_count_arr"}'
upsert_flag = True
jo_insert = {}
jo_insert["rid"] = eval(row_id_str)
jo_insert["key"] = "feat_sample_count_arr"
jo_insert["value"] = feat_sample_count_arr
jstr_insert = json.dumps(jo_insert)
ret = query_mongo.upsert_doc_t(mongo_tuples, filter, jstr_insert,
upsert_flag)
print "INFO: Upsert count for feat_sample_count_arr=", ret
# insert failed, save to local
if ret == 0:
# drop old record in mongo
ret = query_mongo.delete_many(mongo_tuples, None, filter)
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
fsca_hs = os.path.join(local_out_dir, row_id_str,
row_id_str + "_feat_sample_count_arr.pkl")
print "WARNING: save feat_sample_count_arr to local"
ml_util.ml_pickle_save(feat_sample_count_arr, fsca_hs)
# save feature data; TBD. not used. ==================================== ============
#libsvm_rdd=seq_featured_rdd.map(lambda x: convert2libsvm(x,label_idx,data_idx,label_arr))
# put hash to the front of each row, assume hash is after label
libsvm_rdd = seq_featured_rdd.map(
lambda x: x[label_idx + 1] + " " + convert2libsvm(
x, label_idx, data_idx, label_arr))
# debug only
#print "libsvm_rdd="
#for i in libsvm_rdd.collect():
# print i
# get rdd statistics info
stats = featured_rdd.map(lambda p: len(p[metadata_count])).stats()
feat_count_max = stats.max()
feat_count_stdev = stats.stdev()
feat_count_mean = stats.mean()
sample_count = stats.count()
print "INFO: libsvm data: sample count=", sample_count, ",Feat count mean=", feat_count_mean, ",Stdev=", feat_count_stdev
print "INFO: ,max feature count=", feat_count_max
# find sample count
lbl_arr = featured_rdd.map(lambda x: (x[label_idx], 1)).reduceByKey(
add).collect()
print "INFO: Sample count by label=", lbl_arr
# remove duplicated libsvm string; only keep the first duplicated item, assume space following key_idx
if remove_duplicated == "Y":
libsvm_rdd=libsvm_rdd \
.map(lambda x: ( ','.join(x.split(' ')[metadata_count:]), x)) \
.groupByKey().map(lambda x: list(x[1])[0] ) \
.cache()
cnt_list = libsvm_rdd.map(lambda x: (x.split(' ')[1], 1)).reduceByKey(
add).collect()
stats = libsvm_rdd.map(
lambda x: len(x.split(' ')[metadata_count:])).stats()
feat_count_max = stats.max()
feat_count_stdev = stats.stdev()
feat_count_mean = stats.mean()
sample_count = stats.count()
print "INFO: Non-Duplicated libsvm data: sample count=", sample_count, ",Feat count mean=", feat_count_mean, ",Stdev=", feat_count_stdev
print "INFO: ,max feature count=", feat_count_max
print "INFO: Non-Duplicated Label count list=", cnt_list
# clean up libsvm data ==================================== ============
libsvm_data_file = os.path.join(hdfs_folder,
libsvm_alldata_filename) #"libsvm_data"
print "INFO: libsvm_data_file=", libsvm_data_file
try:
#hdfs.ls(save_dir)
#print "find hdfs folder"
hdfs.rmr(libsvm_data_file)
#if num_gram == 1:
# hdfs.rmr(dnn_data_file)
#print "all files removed"
except IOError as e:
print "WARNING: I/O error({0}): {1} at libsvm_data_file clean up".format(
e.errno, e.strerror)
except:
print "WARNING: Unexpected error at libsvm file clean up:", sys.exc_info(
)[0]
#codec = "org.apache.hadoop.io.compress.GzipCodec"
#libsvm_rdd.saveAsTextFile(libsvm_data_file, codec)
libsvm_rdd.saveAsTextFile(libsvm_data_file) # TBD encrypted
feat_count_file = libsvm_data_file + "_feat_count"
print "INFO: feat_count_file=", feat_count_file
try:
hdfs.rmr(feat_count_file)
except IOError as e:
print "WARNING: I/O error({0}): {1} at feat_count clean up".format(
e.errno, e.strerror)
except:
print "WARNING: Unexpected error at libsvm feature count clean up:", sys.exc_info(
)[0]
sc.parallelize([total_feature_numb], 1).saveAsTextFile(feat_count_file)
label_dic = {}
# assign label a number
for idx, label in enumerate(sorted(label_arr)):
if not label in label_dic:
label_dic[
label] = idx #starting from 0, value = idx, e.g., clean:0, dirty:1
# output text for DNN:[meta-data1,meta-data2,..., [feature tokens]] ================= DNN ===========
if num_gram == 1: # special flag to tokenize and keep input orders
print "INFO: processing data for DNN..."
# create token dict
# str_hash_dict: string to hash
# all_hashes_seq_dic: hash to seq id
if token_dict is None or len(token_dict) == 0:
token_dict = {}
str_hash_dict = {v: k for k, v in all_hash_str_dic.iteritems()}
for k, v in str_hash_dict.iteritems():
token_dict[k] = int(all_hashes_seq_dic[str(v)])
#print "token_dict=",len(token_dict),token_dict
dnn_rdd = tmp_rdd \
.map(lambda x: tokenize_by_dict(x, data_idx, token_dict,label_idx, label_dic)) \
.filter(lambda x: len(x) > metadata_count) \
.filter(lambda x: type(x[metadata_count]) is list)
#.cache()
# filter duplication here
#print dnn_rdd.take(3)
dnn_data_file = os.path.join(hdfs_folder,
dnn_alldata_filename) #"dnn_data"
print "INFO: dnn_data_file=", dnn_data_file
try:
hdfs.rmr(dnn_data_file)
except IOError as e:
print "WARNING: I/O error({0}): {1} at dnn_data_file clean up".format(
e.errno, e.strerror)
except:
print "WARNING: Unexpected error at libsvm file clean up:", sys.exc_info(
)[0]
# clean up data
dnn_npy_gz_file = os.path.join(hdfs_folder, row_id_str + "_dnn_")
print "INFO: dnn_npy_gz_file=", dnn_npy_gz_file
try:
hdfs.rmr(dnn_npy_gz_file + "data.npy.gz")
hdfs.rmr(dnn_npy_gz_file + "label.npy.gz")
hdfs.rmr(dnn_npy_gz_file + "info.npy.gz")
except IOError as e:
print "WARNING: I/O error({0}): {1} at dnn_npy clean up".format(
e.errno, e.strerror)
except:
print "WARNING: Unexpected error at dnn_npy file clean up:", sys.exc_info(
)[0]
# save new data
try:
dnn_rdd.saveAsTextFile(dnn_data_file)
except:
print "WARNING: Unexpected error at saving dnn data:", sys.exc_info(
)[0]
# show data statistics
try:
stats = dnn_rdd.map(lambda p: len(p[metadata_count])).stats()
feat_count_max = stats.max()
feat_count_stdev = stats.stdev()
feat_count_mean = stats.mean()
sample_count = stats.count()
print "INFO: DNN data: sample count=", sample_count, ",Feat count mean=", feat_count_mean, ",Stdev=", feat_count_stdev
print "INFO: ,max feature count=", feat_count_max
except:
print "WARNING: Unexpected error at getting stats of dnn_rdd:", sys.exc_info(
)[0]
# clean up pca data in hdfs ============ ========================
pca_files = '*' + libsvm_alldata_filename + "_pca_*"
#print "INFO: pca_files=", pca_files
try:
f_list = hdfs.ls(hdfs_folder)
if len(f_list) > 0:
df_list = fnmatch.filter(f_list, pca_files)
for f in df_list:
print "INFO: rm ", f
hdfs.rmr(f)
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(e.errno, e.strerror)
except:
print "WARNING: Unexpected error at libsvm pca file clean up:", sys.exc_info(
)[0]
# clean up pca data in web local ============ ========================
pca_fname = os.path.join(model_data_folder, row_id_str + '_pca_*.pkl*')
print "INFO: pca_fname=", pca_fname
try:
for fl in glob.glob(pca_fname):
print "INFO: remove ", fl
os.remove(fl)
except OSError, e:
print("Error: %s - %s." % (e.pca_fname, e.strerror))
def train(row_id_str, ds_id, hdfs_feat_dir, local_out_dir, ml_opts_jstr,
excluded_feat_cslist, sp_master, spark_rdd_compress,
spark_driver_maxResultSize, sp_exe_memory, sp_core_max, zipout_dir,
zipcode_dir, zip_file_name, mongo_tuples, labelnameflag, fromweb,
training_fraction, jobname):
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
t0 = time()
t00 = t0
# check if ml_opts.has_excluded_feat ==1 ===================================
has_excluded_feat = 0
ml_opts = {}
if not ml_opts_jstr is None:
ml_opts = json.loads(ml_opts_jstr)
if "has_excluded_feat" in ml_opts:
has_excluded_feat = ml_opts["has_excluded_feat"]
#print "has_excluded_feat=",has_excluded_feat,",excluded_feat_cslist=",excluded_feat_cslist
# get excluded feature list from mongo ========== ===
if str(has_excluded_feat) == "1" and excluded_feat_cslist is None:
excluded_feat_cslist = ml_util.ml_get_excluded_feat(
row_id_str, mongo_tuples)
print "INFO: excluded_feat_cslist=", excluded_feat_cslist
### generate Labeled point
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file:", libsvm_data_file
# load feature count file
feat_count_file = libsvm_data_file + "_feat_count"
feature_count = zip_feature_util.get_feature_count(sc, feat_count_file)
print "INFO: feature_count=", feature_count
# load sample RDD from text file
# also exclude selected features in sample ================ =====
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(
sc, libsvm_data_file, feature_count, excluded_feat_cslist)
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
# get distinct label list
labels_list_all = samples_rdd.map(
lambda p: p[0].label).distinct().collect()
### generate training and testing data
training_rdd, testing_rdd = samples_rdd.randomSplit(
[training_fraction, 1 - training_fraction])
training_rdd = training_rdd.map(lambda p: p[0]) # keep LabeledPoint only
training_rdd.cache()
training_sample_count = training_rdd.count()
training_lbl_cnt_list = training_rdd.map(
lambda p: (p.label, 1)).reduceByKey(add).collect()
testing_rdd.cache()
testing_sample_count = testing_rdd.count()
testing_lbl_cnt_list = testing_rdd.map(
lambda p: (p[0].label, 1)).reduceByKey(add).collect()
sample_count = training_sample_count + testing_sample_count
t1 = time()
print "INFO: training sample count=", training_sample_count, ", testing sample count=", testing_sample_count
print "INFO: training label list=", training_lbl_cnt_list, ", testing label list=", testing_lbl_cnt_list
print "INFO: labels_list_all=", labels_list_all
print "INFO: training and testing samples generated!"
print 'INFO: running time: %f' % (t1 - t0)
t0 = t1
##############################################
########### Grid Search with CV ##############
##############################################
### get the parameters for cross validation and grid search ###
(cv, model_name, param_dict) = generate_param(ml_opts)
### generate label names (family names) #####
### connect to database to get the column list which contains all column number of the corresponding feature####
if labelnameflag == 1:
label_dic = ml_util.ml_get_label_dict(row_id_str, mongo_tuples, ds_id)
print "INFO: label_dic:", label_dic
else:
label_dic = {}
label_set = set(labels_list_all)
for label_value in label_set:
label_dic[int(label_value)] = str(int(label_value))
print "INFO: generated label_dic:", label_dic
labels_list = []
for key in sorted(label_dic):
labels_list.append(label_dic[key])
#print "labels:", labels_list
class_num = len(labels_list)
if class_num > 2:
print "INFO: Multi-class classification! Number of classes = ", class_num
#### generate training and testing rdd(s) for CV#####
split_prob = 1.0 / float(cv)
split_prob_list = []
for i in range(0, cv):
split_prob_list.append(split_prob)
list_rdd = training_rdd.randomSplit(split_prob_list)
list_train_rdd = []
list_test_rdd = []
for i in range(0, cv):
list_rdd[i].cache()
for i in range(0, cv):
tr_rdd = sc.emptyRDD()
for j in range(0, cv):
if j == i:
pass
else:
tr_rdd = tr_rdd + list_rdd[j]
tr_rdd.cache()
list_train_rdd.append(tr_rdd)
list_test_rdd.append(list_rdd[i])
all_comb_list_of_dic = get_all_combination_list_of_dic(param_dict)
print "INFO: Total number of searching combinations:", len(
all_comb_list_of_dic)
### loop for all parameter combinations and search the best parameters with CV###
results = []
for p in range(0, len(all_comb_list_of_dic)):
params = all_comb_list_of_dic[p]
C = params['C']
iteration_num = params['iterations']
regularization = params['regType']
scores = []
for i in range(0, cv):
train_rdd = list_train_rdd[i]
test_rdd = list_test_rdd[i]
train_number = train_rdd.count()
regP = C / float(train_number)
### build model ###
if model_name == "linear_svm_with_sgd":
#print "====================1: Linear SVM============="
model_classification = SVMWithSGD.train(
train_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization) # regParam = 1/(sample_number*C)
elif model_name == "logistic_regression_with_lbfgs":
#print "====================2: LogisticRegressionWithLBFGS============="
model_classification = LogisticRegressionWithLBFGS.train(
train_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization,
numClasses=class_num) # regParam = 1/(sample_number*C)
elif model_name == "logistic_regression_with_sgd":
#print "====================3: LogisticRegressionWithSGD============="
model_classification = LogisticRegressionWithSGD.train(
train_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization) # regParam = 1/(sample_number*C)
else:
print "ERROR: Training model selection error: no valid ML model selected!"
return
### Evaluating the model on testing data
labelsAndPreds = test_rdd.map(
lambda p: (p.label, model_classification.predict(p.features)))
labelsAndPreds.cache()
test_sample_number = test_rdd.count()
testErr = labelsAndPreds.filter(
lambda (v, p): v != p).count() / float(test_sample_number)
accuracy = 1 - testErr
#print "Accuracy = ", accuracy
scores.append(accuracy)
ss = np.asarray(scores)
#print "%0.3f (+/-%0.03f) for " % (ss.mean(), ss.std() * 2), params
results.append((ss.mean(), ss.std() * 2, params))
sorted_results = sorted(results, key=lambda x: x[0], reverse=1)
(best_accuracy, best_std2, best_param) = sorted_results[0]
print "INFO: ml_opts_jstr=", ml_opts_jstr
print "INFO: best_param=", best_param
#ml_opts=json.loads(ml_opts_jstr);
print "INFO: ml_opts=", ml_opts
##############################################
######output Grid Search results##############
##############################################
json2save = {}
json2save["rid"] = int(row_id_str)
json2save["key"] = "cv_result"
#json2save["param_str"]=ml_opts_jstr
json2save["param_dic"] = param_dict
cv_grid = []
print ""
print "INFO: =====Grid Search Results for SPARK ======"
print "INFO: Best parameters set found for ", model_name, " is: "
for key in best_param:
print "INFO:", key, "=", best_param[key]
if key.lower() == "regtype":
ml_opts['regularization'] = str(best_param[key])
else:
ml_opts[key.lower()] = str(best_param[key]) # add best param to
ml_opts_jstr = json.dumps(ml_opts)
json2save["param_str"] = ml_opts_jstr
print "INFO: Average accuracy with CV = ", cv, ": ", best_accuracy
print ""
print "INFO: Grid scores on development set:"
for i in range(0, len(sorted_results)):
(ave_accu_i, std2_i, param_i) = sorted_results[i]
print "%0.3f (+/-%0.03f) for " % (ave_accu_i, std2_i), param_i
#outstr='%s,%0.3f,%0.03f,%s' % (param_i,ave_accu_i, std2_i,"Selected" if param_i==best_param else "")
outj = {}
outj["param"] = param_i
outj["average_accuracy"] = "%0.3f" % (ave_accu_i)
outj["std_deviation"] = "%0.3f" % (std2_i)
outj["selected"] = "%s" % ("Selected" if param_i == best_param else "")
cv_grid.append(outj)
print " "
t1 = time()
print 'INFO: Grid Search with CV run time: %f' % (t1 - t0)
t0 = time()
##################################################################################
json2save["cv_grid_data"] = cv_grid
cv_result = json.dumps(json2save)
print "INFO: cv_result=", cv_result
filter = '{"rid":' + row_id_str + ',"key":"cv_result"}'
upsert_flag = True
## write to mongoDB.myml.dataset_info, ignore doc with duplicated key
# db.dataset_info.createIndex({"rid":1,"key":1},{unique:true})
ret = query_mongo.upsert_doc_t(mongo_tuples, filter, cv_result,
upsert_flag)
print "INFO: Upsert count for mllib cv_result: ret=", ret
############################################################################################
########### retrain with all training data and generate the final model with results #######
############################################################################################
C = best_param['C']
iteration_num = best_param['iterations']
regularization = best_param['regType']
regP = C / float(training_sample_count)
######################################the rest of the code is the same as train_MLlib.py #####################################################################
if model_name == "linear_svm_with_sgd":
### 1: linearSVM
print "INFO: ====================1: Linear SVM============="
model_classification = SVMWithSGD.train(
training_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization) # regParam = 1/(sample_number*C)
#print model_classification
elif model_name == "logistic_regression_with_lbfgs":
### 2: LogisticRegressionWithLBFGS
print "INFO: ====================2: LogisticRegressionWithLBFGS============="
model_classification = LogisticRegressionWithLBFGS.train(
training_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization,
numClasses=class_num) # regParam = 1/(sample_number*C)
elif model_name == "logistic_regression_with_sgd":
### 3: LogisticRegressionWithSGD
print "INFO: ====================3: LogisticRegressionWithSGD============="
model_classification = LogisticRegressionWithSGD.train(
training_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization) # regParam = 1/(sample_number*C)
else:
print "INFO: Training model selection error: no valid ML model selected!"
return
print "INFO: model type=", type(model_classification)
# create feature coefficient file ================================
coef_arr = None
intercept = None
if model_classification.weights is None:
print "WARNING: model weights not found!"
else:
coef_arr = model_classification.weights.toArray().tolist()
# save to mongo
key = "coef_arr"
ret = ml_util.save_json_t(row_id_str, key, coef_arr, mongo_tuples)
# save intercept to mongo
key = "coef_intercept"
intercept = model_classification.intercept
ret = ml_util.save_json_t(row_id_str, key, intercept, mongo_tuples)
# feature list + coef file =============
feat_filename = os.path.join(local_out_dir,
row_id_str + "_feat_coef.json")
print "INFO: feat_filename=", feat_filename
# create feature list + coef file =============================================== ============
# expect a dict of {"fid":(coef, feature_raw_string)}
jret = ml_util.build_feat_list_t(row_id_str, feat_filename, None, None,
coef_arr, ds_id, mongo_tuples)
# special featuring for IN or libsvm
if jret is None:
jret = ml_util.build_feat_coef_raw_list_t(row_id_str,
feat_filename, coef_arr,
ds_id, mongo_tuples)
if jret is None:
print "WARNING: Cannot create sample list for testing dataset. "
jfeat_coef_dict = jret
print "INFO: coef_arr len=", len(
coef_arr), ", feature_count=", feature_count
# for multi-class
if len(coef_arr) != feature_count:
jfeat_coef_dict = {}
print "WARNING: feature list can't be shown for multi-class classification"
# Calculate prediction and Save testing dataset
bt_coef_arr = sc.broadcast(coef_arr)
bt_intercept = sc.broadcast(intercept)
bt_jfeat_coef_dict = sc.broadcast(jfeat_coef_dict)
### Evaluating the model on testing dataset: label, predict label, score, feature list
print "INFO: intercept=", intercept
print "INFO: coef_arr len=", len(coef_arr)
print "INFO: jfeat_coef_dict len=", len(jfeat_coef_dict)
# get prediction of testing dataset : (tlabel, plabel, score, libsvm, raw feat str, hash) ==============================
if len(coef_arr) == feature_count:
testing_pred_rdd = testing_rdd.map(lambda p: (
p[0].label \
,model_classification.predict(p[0].features) \
,zip_feature_util.calculate_hypothesis(p[0].features, bt_coef_arr.value, bt_intercept.value, model_name) \
,p[0].features \
,p[1] \
) ).cache()
else: # for multi-class, no prediction score;, TBD for better solution: how to display multiple weights for each class
testing_pred_rdd = testing_rdd.map(lambda p: (
p[0].label \
,model_classification.predict(p[0].features) \
,0 \
,p[0].features \
,p[1] \
) ).cache()
# save false prediction to local file
false_pred_fname = os.path.join(local_out_dir,
row_id_str + "_false_pred.json")
print "INFO: false_pred_fname=", false_pred_fname
false_pred_data=testing_pred_rdd.filter(lambda p: p[0] != p[1])\
.map(lambda p: (p[0],p[1],p[2] \
,zip_feature_util.get_dict_coef_raw4feat(zip_feature_util.sparseVector2dict(p[3]), bt_jfeat_coef_dict.value)
,p[4] ) ) \
.collect()
print "INFO: false predicted count=", len(false_pred_data)
false_pred_arr = []
with open(false_pred_fname, "w") as fp:
for sp in false_pred_data:
jsp = {
"tlabel": sp[0],
"plabel": sp[1],
"score": sp[2],
"feat": sp[3],
"hash": sp[4]
}
#print "jsp=",jsp
false_pred_arr.append(jsp)
fp.write(json.dumps(false_pred_arr))
# save prediction results, format: label, prediction, hash
pred_ofname = os.path.join(local_out_dir,
row_id_str + "_pred_output.pkl")
print "INFO: pred_ofname=", pred_ofname
pred_out_arr = testing_pred_rdd.map(lambda p:
(p[0], p[1], p[4])).collect()
ml_util.ml_pickle_save(pred_out_arr, pred_ofname)
### Evaluating the model on testing data
#labelsAndPreds = testing_rdd.map(lambda p: (p.label, model_classification.predict(p.features)))
labelsAndPreds = testing_pred_rdd.map(lambda p: (p[0], p[1]))
labelsAndPreds.cache()
#testing_sample_count = testing_rdd.count()
testErr = labelsAndPreds.filter(lambda (v, p): v != p).count() / float(
testing_sample_count)
accuracy = 1 - testErr
print "INFO: Accuracy = ", accuracy
### Save model
#save_dir = config.get('app', 'HADOOP_MASTER')+'/user/hadoop/yigai/row_6/'
#save_dir = config.get('app', 'HADOOP_MASTER')+config.get('app', 'HDFS_MODEL_DIR')+'/'+row_id_str
save_dir = os.path.join(config.get('app', 'HADOOP_MASTER'),
config.get('app', 'HDFS_MODEL_DIR'), row_id_str)
try:
hdfs.ls(save_dir)
#print "find hdfs folder"
hdfs.rmr(save_dir)
#print "all files removed"
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(
e.errno, e.strerror), ". At HDFS=", save_dir
except:
print "WARNING: Unexpected error:", sys.exc_info(
)[0], ". At HDFS=", save_dir
model_classification.save(sc, save_dir)
###load model if needed
#sameModel = SVMModel.load(sc, save_dir)
t1 = time()
print 'INFO: training run time: %f' % (t1 - t0)
t0 = t1
###############################################
###########plot prediction result figure#######
###############################################
labels = labelsAndPreds.collect()
true_label_list = [x for x, _ in labels]
pred_label_list = [x for _, x in labels]
pred_fname = os.path.join(local_out_dir, row_id_str + "_1" + ".png")
true_fname = os.path.join(local_out_dir, row_id_str + "_2" + ".png")
pred_xlabel = 'Prediction (Single Run)'
true_xlabel = 'True Labels (Single Run)'
test_cnt_dic = ml_util.ml_plot_predict_figures(
pred_label_list, true_label_list, labels_list, label_dic,
testing_sample_count, pred_xlabel, pred_fname, true_xlabel, true_fname)
plt.show()
perf_measures = None
dataset_info = {
"training_fraction": training_fraction,
"class_count": class_num,
"dataset_count": sample_count
}
#############################################################
###################for 2 class only (plot ROC curve)#########
#############################################################
if len(labels_list) == 2:
do_ROC = True
reverse_label_dic = dict((v, k) for k, v in label_dic.items())
if 'clean' in reverse_label_dic:
flag_clean = reverse_label_dic['clean']
elif 'benign' in reverse_label_dic:
flag_clean = reverse_label_dic['benign']
elif '0' in reverse_label_dic:
flag_clean = 0
else:
print "WARNING: No ROC curve generated: 'clean' or '0' must be a label for indicating negative class!"
do_ROC = False
# build data file for score graph
score_graph_fname = os.path.join(local_out_dir,
row_id_str + "_score_graph.json")
print "INFO: score_graph_fname=", score_graph_fname
# build score_arr_0, score_arr_1
# format: tlabel, plabel, score, libsvm, raw feat str, hash
graph_arr = testing_pred_rdd.map(lambda p:
(int(p[0]), float(p[2]))).collect()
score_arr_0 = []
score_arr_1 = []
max_score = 0
min_score = 0
for p in graph_arr:
if p[0] == 0:
score_arr_0.append(p[1])
else:
score_arr_1.append(p[1])
# save max,min score
if p[1] > max_score:
max_score = p[1]
elif p[1] < min_score:
min_score = p[1]
ml_build_pred_score_graph(score_arr_0, score_arr_1, model_name,
score_graph_fname, max_score, min_score)
#print "score_arr_0=",score_arr_0
#print "score_arr_1=",score_arr_1
#print "max_score=",max_score
#print "min_score=",min_score
if do_ROC:
perf_measures = ml_util.calculate_fscore(true_label_list,
pred_label_list)
print "RESULT: perf_measures=", perf_measures
model_classification.clearThreshold()
scoreAndLabels = testing_rdd.map(lambda p: (
model_classification.predict(p[0].features), int(p[0].label)))
#metrics = BinaryClassificationMetrics(scoreAndLabels)
#areROC = metrics.areaUnderROC
#print areROC
scoreAndLabels_list = scoreAndLabels.collect()
if flag_clean == 0:
scores = [x for x, _ in scoreAndLabels_list]
s_labels = [x for _, x in scoreAndLabels_list]
testing_N = test_cnt_dic[0]
testing_P = test_cnt_dic[1]
else:
scores = [-x for x, _ in scoreAndLabels_list]
s_labels = [1 - x for _, x in scoreAndLabels_list]
testing_N = test_cnt_dic[1]
testing_P = test_cnt_dic[0]
#print scores
#print s_labels
# create ROC data file ======== ====
roc_auc = ml_create_roc_files(row_id_str, scores, s_labels,
testing_N, testing_P, local_out_dir,
row_id_str)
perf_measures["roc_auc"] = roc_auc
# only update db for web request
if fromweb == "1":
#print "database update"
str_sql="UPDATE atdml_document set "+"accuracy = '"+str(accuracy*100)+"%" \
+"', status = 'learned', processed_date ='"+str(datetime.datetime.now()) \
+"',ml_opts='"+ml_opts_jstr \
+"', perf_measures='"+json.dumps(perf_measures) \
+"', dataset_info='"+json.dumps(dataset_info) \
+"' where id="+row_id_str
ret = exec_sqlite.exec_sql(str_sql)
print "INFO: Data update done! ret=", str(ret)
else:
print "INFO: accuracy = '" + str(accuracy * 100) + "%"
t1 = time()
print 'INFO: total run time: %f' % (t1 - t00)
print 'INFO: Finished!'
return 0
def feat_importance_2way(row_id_str, ds_id, hdfs_feat_dir, score_file_IT,
score_file_prob, sp_master, spark_rdd_compress,
spark_driver_maxResultSize, sp_exe_memory,
sp_core_max, zipout_dir, zipcode_dir, zip_file_name,
mongo_tuples, jobname, uploadtype, description_file):
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_util.ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
t0 = time()
# get folder list (labels) from hdfs data_out/<id>/metadata
dirFile_loc = os.path.join(hdfs_feat_dir, "metadata")
dirFolders = sc.textFile(dirFile_loc)
hash_Folders = dirFolders.collect()
#print "hash_Folders=",hash_Folders
folder_list = [x.encode('UTF8') for x in hash_Folders]
#print "INFO: dirFile_loc=",dirFile_loc,", folder_list=",folder_list
row_num_training = 0
sample_numbers = []
sparse_mtx_list = []
features_list = []
row_list = []
col_list = []
max_feat_list = []
for folder in folder_list:
print "INFO: folder:", folder
label = folder_list.index(folder) + 1
print 'INFO: label=', label
# md5 list
logFile_name = os.path.join(hdfs_feat_dir, folder, mtx_name_list)
# libsvm data
logFile_data = os.path.join(hdfs_feat_dir, folder, mtx_libsvm)
#logFile_data = hdfs_feat_dir + folder + mtx_stat
#print "INFO: logFile_name=",logFile_name
#print "INFO: logFile_data=",logFile_data
logNames = sc.textFile(logFile_name).cache()
logData = sc.textFile(logFile_data).cache()
names = logNames.collect()
data = logData.collect()
name_l = [x.encode('UTF8') for x in names]
feature_l = [x.encode('UTF8') for x in data]
name_list = [names.strip() for names in name_l]
feature_list = [features.strip() for features in feature_l]
num_names = len(name_list)
sample_numbers.append(num_names)
print 'INFO: sample_numbers=', sample_numbers
print 'INFO: name_list count=', num_names
########generate list for csc_matrix creation #########
i = 0
features_training = []
row_training = []
col_training = []
labels_training = []
max_feat = 0
while i < num_names:
features = feature_list[i]
features = features.strip()
feature_array = features.split(' ')
labels_training.append(label)
length = len(feature_array)
j = 0
while j < length:
feature = feature_array[j]
feat, value = feature.split(':', 2)
row_training.append(i)
col_training.append(int(feat) - 1)
features_training.append(int(value))
max_feat = max(max_feat, int(feat))
j = j + 1
i = i + 1
#print "features_training=",features_training
#print "row_training=",row_training
#print "col_training=",col_training
features_training = array(features_training)
row_training = array(row_training)
col_training = array(col_training)
#print "type(row_training)=",type(row_training),",row_training=",row_training
#print "type(col_training)=",type(col_training),",col_training=",col_training
max_feat_list.append(max_feat)
features_list.append(features_training)
row_list.append(row_training)
col_list.append(col_training)
row_num_training = row_num_training + num_names
#print " END for folder in folder_list: "
col_num = max(max_feat_list)
print "INFO: column number=", col_num, ", len(max_feat_list)=", len(
max_feat_list)
for i in range(0, len(max_feat_list)):
sparse_mtx = csc_matrix((features_list[i], (row_list[i], col_list[i])),
shape=(sample_numbers[i], col_num))
sparse_mtx_list.append(sparse_mtx)
#print sparse_mtx_list[0]
#print "sparse_mtx_list[0].shape=",sparse_mtx_list[0].shape
#print sparse_mtx_list[1]
#print "sparse_mtx_list[1].shape=",sparse_mtx_list[1].shape
exclusive_feature_set_mal = []
exclusive_feature_set_clean = []
dic_feature_cnt_mal = {}
dic_feature_cnt_clean = {}
dic_score = {}
dic_cnt_mal = {}
dic_cnt_clean = {}
dic_IT_grain = {}
####################################################
####feature importance algorithms: 2 methods ####### # Only for 2 classes ???
####################################################
if len(sample_numbers) == 2:
###################################################
################## calculate probability ############
###################################################
print "INFO: ======= Feature Importance(probability) ================"
for j in range(0, col_num):
curr_col_dirty = sparse_mtx_list[0].getcol(j)
sum_col = curr_col_dirty.sum(0)
cnt_mal = sum_col.tolist()[0][0]
curr_col_clean = sparse_mtx_list[1].getcol(j)
sum_col = curr_col_clean.sum(0)
cnt_clean = sum_col.tolist()[0][0]
percnt_mal = cnt_mal / float(sample_numbers[0])
percnt_clean = cnt_clean / float(sample_numbers[1])
score_j = (percnt_mal + 1 - percnt_clean) / 2
dic_score[j + 1] = score_j
dic_cnt_clean[j + 1] = cnt_clean
dic_cnt_mal[j + 1] = cnt_mal
sorted_score = sorted(dic_score.items(),
key=operator.itemgetter(1),
reverse=True)
#print "sorted_score:", sorted_score
#print "dic_cnt_clean", dic_cnt_clean
#print "dic_cnt_mal", dic_cnt_mal
if os.path.exists(score_file_prob):
try:
os.remove(score_file_prob)
except OSError, e:
print("Error: %s - %s." % (e.score_file_prob, e.strerror))
for ii in range(0, len(sorted_score)):
(feat, score) = sorted_score[ii]
#print feat, score, dic_all_columns[feat]
str01 = str(feat) + "\t" + str(
score) + "\t" + description_str + "\n"
with open(score_file_prob, "a") as f:
f.write(str01)
########################################################
##################Information Gain (entropy)############
########################################################
print "INFO: ======= Information Gain ================"
for j in range(0, col_num):
cnt_mal = dic_cnt_mal[j + 1]
cnt_clean = dic_cnt_clean[j + 1]
total_samples = sample_numbers[0] + sample_numbers[1]
p0 = float(sample_numbers[0]) / total_samples
p1 = 1 - p0
if p0 == 0 or p1 == 0:
parent_entropy = 0
else:
parent_entropy = 0 - p0 * np.log2(p0) - p1 * np.log2(p1)
if cnt_clean + cnt_mal == 0:
information_gain = 0
elif total_samples - cnt_clean - cnt_mal == 0:
information_gain = 0
else:
p0 = float(cnt_mal) / (cnt_clean + cnt_mal)
p1 = 1 - p0
if p0 == 0 or p1 == 0:
child_left_entropy = 0
else:
child_left_entropy = 0 - p0 * np.log2(p0) - p1 * np.log2(
p1)
p0 = float(sample_numbers[0] - cnt_mal) / (total_samples -
cnt_clean - cnt_mal)
p1 = 1 - p0
if p0 == 0 or p1 == 0:
child_right_entropy = 0
else:
child_right_entropy = 0 - p0 * np.log2(p0) - p1 * np.log2(
p1)
weighted_child_entropy = child_left_entropy * float(
cnt_clean +
cnt_mal) / total_samples + child_right_entropy * float(
total_samples - cnt_clean - cnt_mal) / total_samples
information_gain = parent_entropy - weighted_child_entropy
dic_IT_grain[j + 1] = information_gain
sorted_IT_gain = sorted(dic_IT_grain.items(),
key=operator.itemgetter(1),
reverse=True)
if os.path.exists(score_file_IT):
try:
os.remove(score_file_IT)
except OSError, e:
print("ERROR: %s - %s." % (e.score_file_IT, e.strerror))
def pca(row_id_str, ds_id, hdfs_feat_dir, local_out_dir
, sp_master, spark_rdd_compress, spark_driver_maxResultSize, sp_exe_memory, sp_core_max
, zipout_dir, zipcode_dir, zip_file_name
, mongo_tuples, fromweb, pca_jstr
, jobname, model_data_folder ):
# create zip files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir, zipcode_dir, zip_file_name, prefix='zip_feature_util')
print "INFO: zip_file_path=",zip_file_path
# init Spark context ====
sc=ml_util.ml_get_spark_context(sp_master
, spark_rdd_compress
, spark_driver_maxResultSize
, sp_exe_memory
, sp_core_max
, jobname
, [zip_file_path])
pca_param=json.loads(pca_jstr)
if "k" in pca_param:
k=pca_param["k"]
else:
k=None
if "threshold" in pca_param:
threshold=pca_param["threshold"]
else:
threshold=None
if "lib" in pca_param:
lib=pca_param["lib"]
else:
lib='mllib'
ret=-1
# start here =================================================================== ===============
t0 = time()
# source libsvm filename
libsvm_data_file = os.path.join(hdfs_feat_dir , "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
# load sample RDD from text file
# format Row(label, features, hash) from get_sample_dataframe()
samples_df, feature_count = zip_feature_util.get_sample_dataframe(sc, libsvm_data_file, 0, None)
print "INFO: feature_count=",feature_count
#df_pcaed format: hash,label, features
(df_pcaed, k, pca_model)=PCA_transform(sc, samples_df,feature_count, threshold, k)
print "INFO: Doing PCA... threshold=",threshold,",k=",k
#print "df_pcaed=",df_pcaed.first()
#print "k=",k
#print "pca_model=",pca_model
#print "pc=",pca_model.pc
# pca model filename ============================= ===============
if model_data_folder is None:
if row_id_str != ds_id:
# get from parent dataset
model_data_folder = os.path.join(config.get('app', 'HADOOP_MASTER'),config.get('app', 'HDFS_MODEL_DIR'), ds_id+"_pca")
else:
model_data_folder = os.path.join(config.get('app', 'HADOOP_MASTER'),config.get('app', 'HDFS_MODEL_DIR'), row_id_str+"_pca")
# create HDFS folder
try:
hdfs.mkdir(model_data_folder)
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(e.errno, e.strerror),". At HDFS=", save_dir
except:
print "WARNING: Unexpected error:", sys.exc_info()[0] ,". At HDFS=", save_dir
if not threshold is None:
#pca_fname=os.path.join(hdfs_feat_dir , row_id_str+'_pca_'+str(threshold)+'.ml')
pca_fname=os.path.join(model_data_folder , 'pca_model_'+str(threshold))
libsvm_data_pca = os.path.join(hdfs_feat_dir , "libsvm_data_pca_"+str(threshold)+'.ml')
else:
pca_fname=os.path.join(model_data_folder , 'pca_model_'+str(k))
libsvm_data_pca = os.path.join(hdfs_feat_dir , "libsvm_data_pca_"+str(k)+'.ml')
# save pca model to HDFS ===============
print "INFO: pca_fname=",pca_fname
pca_model.write().overwrite().save(pca_fname)
# save pca data to HDFS ============================= ===============
print "INFO: libsvm_data_pca=",libsvm_data_pca
# construct libsvm string
libsvm_rdd=df_pcaed.rdd.map(lambda p: p[0]+" "+str(int(p[1]))+zip_feature_util.dv2libsvm(p[2].toArray()))
# clean up old libsvm file ============================= ===============
try:
hdfs.rmr(libsvm_data_pca)
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(e.errno, e.strerror)
except:
print "WARNING: Unexpected error at rmr():", sys.exc_info()[0]
# overwrite pca file at hdfs
libsvm_rdd.saveAsTextFile(libsvm_data_pca)
t1 = time()
print 'INFO: PCA processing time: %f' %(t1-t0)
### insert pca_param into mongoDB ###
filter='{"rid":'+row_id_str+',"key":"pca_param"}'
if not threshold is None:
pca_param["threshold"]=threshold
if not k is None:
pca_param["k"]=k
print "INFO: pca_param=",pca_param
upsert_flag=True
jstr_insert = '{ "rid":'+row_id_str+',"key":"pca_param", "value":'+json.dumps(pca_param)+'}'
ret=query_mongo.upsert_doc_t(mongo_tuples,filter,jstr_insert,upsert_flag)
print "INFO: Upsert count for pca_param=",ret
# only update db for web request ===========
if fromweb=="1":
#print "database update"
str_sql="UPDATE atdml_document set " \
+" status = 'pca-ed', processed_date ='"+str(datetime.datetime.now()) \
+"' , ml_pca_opts = '"+json.dumps(pca_param) \
+"' where id="+row_id_str
ret=exec_sqlite.exec_sql(str_sql)
print "INFO: Update Sqlite DB done! ret=", str(ret)
t1 = time()
print 'INFO: running time: %f' %(t1-t0)
#print 'Finished!'
return 0
def train(row_id_str,
ds_id,
hdfs_feat_dir,
local_out_dir,
ml_opts_jstr,
excluded_feat_cslist,
sp_master,
spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory,
sp_core_max,
zipout_dir,
zipcode_dir,
zip_file_name,
mongo_tuples,
labelnameflag,
fromweb,
training_fraction,
jobname,
random_seed=None):
### generate data folder and out folder, clean up if needed
#local_out_dir = local_out_dir + "/"
#if os.path.exists(local_out_dir):
# shutil.rmtree(local_out_dir) # to keep smaplelist file
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
# create zip files for Spark workers ================= ================
zip_file_path = ml_util.ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
t0 = time()
# check if ml_opts.has_excluded_feat ==1 ===================================
has_excluded_feat = 0
ml_opts = {}
if not ml_opts_jstr is None:
ml_opts = json.loads(ml_opts_jstr)
if "has_excluded_feat" in ml_opts:
has_excluded_feat = ml_opts["has_excluded_feat"]
#print "has_excluded_feat=",has_excluded_feat,",excluded_feat_cslist=",excluded_feat_cslist
# get excluded feature list from mongo ========== ===
if str(has_excluded_feat) == "1" and excluded_feat_cslist is None:
excluded_feat_cslist = ml_util.ml_get_excluded_feat(
row_id_str, mongo_tuples)
print "INFO: excluded_feat_cslist=", excluded_feat_cslist
# filename for featured data
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file:", libsvm_data_file
# load feature count file
feat_count_file = libsvm_data_file + "_feat_count"
feature_count = zip_feature_util.get_feature_count(sc, feat_count_file)
print "INFO: feature_count=", feature_count
# load sample RDD from text file
# also exclude selected features in sample ================ =====
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(
sc, libsvm_data_file, feature_count, excluded_feat_cslist)
# get distinct label list
labels_list_all = samples_rdd.map(
lambda p: p[0].label).distinct().collect()
# split samples to training and testing data, format (LabeledPoint,hash)
training_rdd, testing_rdd = samples_rdd.randomSplit(
[training_fraction, 1 - training_fraction], seed=int(random_seed))
training_rdd = training_rdd.map(lambda p: p[0]) # keep LabeledPoint only
training_rdd.cache()
training_sample_count = training_rdd.count()
training_lbl_cnt_list = training_rdd.map(
lambda p: (p.label, 1)).reduceByKey(add).collect()
testing_rdd.cache()
testing_sample_count = testing_rdd.count()
testing_lbl_cnt_list = testing_rdd.map(
lambda p: (p[0].label, 1)).reduceByKey(add).collect()
sample_count = training_sample_count + testing_sample_count
# random_seed testing
if not random_seed is None:
all_t = testing_rdd.collect()
all_t = sorted(all_t, key=lambda x: x[1])
cnt = 0
for i in all_t:
print i[1]
cnt = cnt + 1
if cnt > 3:
break
t1 = time()
print "INFO: training sample count=", training_sample_count, ", testing sample count=", testing_sample_count
print "INFO: training label list=", training_lbl_cnt_list, ", testing label list=", testing_lbl_cnt_list
print "INFO: labels_list_all=", labels_list_all
print "INFO: training and testing samples generated!"
print 'INFO: running time: %f' % (t1 - t0)
t0 = t1
###############################################
###########build learning model################
###############################################
### get the parameters###
print "INFO: ======Learning Algorithm and Parameters============="
#ml_opts = json.loads(ml_opts_jstr)
model_name = ml_opts[
'learning_algorithm'] # 1: linear_svm_with_sgd; 2: logistic_regression_with_lbfgs; 3: logistic_regression_with_sgd
iteration_num = 0
if 'iterations' in ml_opts:
iteration_num = ml_opts['iterations']
C = 0
if 'c' in ml_opts:
C = eval(ml_opts['c'])
regularization = ""
if 'regularization' in ml_opts:
regularization = ml_opts['regularization']
print "INFO: Learning Algorithm: ", model_name
print "INFO: C = ", C
print "INFO: iterations = ", iteration_num
print "INFO: regType = ", regularization
regP = C / float(training_sample_count)
print "INFO: Calculated: regParam = ", regP
### generate label names (family names) #####
### connect to database to get the column list which contains all column number of the corresponding feature####
if labelnameflag == 1:
'''
key = "dic_name_label"
jstr_filter='{"rid":'+row_id_str+',"key":"'+key+'"}'
jstr_proj='{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter='{"rid":'+ds_id+',"key":"'+key+'"}'
doc=query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_list = doc['value']
print "INFO: dic_list=",dic_list
label_dic = {}
for i in range(0, len(dic_list)):
for key in dic_list[i]:
label_dic[dic_list[i][key]] = key.encode('UTF8')
'''
label_dic = ml_util.ml_get_label_dict(row_id_str, mongo_tuples, ds_id)
print "INFO: label_dic:", label_dic
else:
label_dic = {}
label_set = set(labels_list_all)
for label_value in label_set:
label_dic[int(label_value)] = str(int(label_value))
print "INFO: generated label_dic:", label_dic
labels_list = []
for key in sorted(label_dic):
labels_list.append(label_dic[key])
print "INFO: labels:", labels_list
class_num = len(labels_list)
if class_num > 2:
print "INFO: Multi-class classification! Number of classes = ", class_num
### build model ###
if model_name == "linear_svm_with_sgd":
### 1: linearSVM
print "INFO: ====================1: Linear SVM============="
model_classification = SVMWithSGD.train(
training_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization) # regParam = 1/(sample_number*C)
#print model_classification
elif model_name == "logistic_regression_with_lbfgs":
### 2: LogisticRegressionWithLBFGS
print "INFO: ====================2: LogisticRegressionWithLBFGS============="
model_classification = LogisticRegressionWithLBFGS.train(
training_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization,
numClasses=class_num) # regParam = 1/(sample_number*C)
elif model_name == "logistic_regression_with_sgd":
### 3: LogisticRegressionWithSGD
print "INFO: ====================3: LogisticRegressionWithSGD============="
model_classification = LogisticRegressionWithSGD.train(
training_rdd,
regParam=regP,
iterations=iteration_num,
regType=regularization) # regParam = 1/(sample_number*C)
else:
print "INFO: Training model selection error: no valid ML model selected!"
return
print "INFO: model type=", type(model_classification)
# create feature coefficient file ================================
coef_arr = None
intercept = None
if model_classification.weights is None:
print "WARNING: model weights not found!"
else:
coef_weights = model_classification.weights
#print "coef_weights=",coef_weights
#print type(coef_weights),coef_weights.shape
coef_arr = coef_weights.toArray().tolist()
# save coef_arr to mongo
key = "coef_arr"
ret = ml_util.save_json_t(row_id_str, key, coef_arr, mongo_tuples)
# save coef_arr to local file
if ret == 0:
# drop old record in mongo
filter = '{"rid":' + row_id_str + ',"key":"coef_arr"}'
ret = query_mongo.delete_many(mongo_tuples, None, filter)
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
fn_ca = os.path.join(local_out_dir, row_id_str,
row_id_str + "_coef_arr.pkl")
print
ml_util.ml_pickle_save(coef_arr, fn_ca)
# save intercept to mongo
intercept = model_classification.intercept
key = "coef_intercept"
ret = ml_util.save_json_t(row_id_str, key, intercept, mongo_tuples)
# feature list + coef file =============
feat_filename = os.path.join(local_out_dir,
row_id_str + "_feat_coef.json")
print "INFO: feat_filename=", feat_filename
# create feature, coef & raw string file =============================================== ============
# expect a dict of {"fid":(coef, feature_raw_string)}
jret = ml_util.build_feat_list_t(row_id_str, feat_filename, None, None,
coef_arr, ds_id, mongo_tuples)
# special featuring for IN or libsvm
if jret is None:
jret = ml_util.build_feat_coef_raw_list_t(row_id_str,
feat_filename, coef_arr,
ds_id, mongo_tuples)
if jret is None:
print "WARNING: Cannot create sample list for testing dataset. "
jfeat_coef_dict = jret
print "INFO: coef_arr len=", len(
coef_arr), ", feature_count=", feature_count
# for multi-class
if len(coef_arr) != feature_count:
jfeat_coef_dict = {}
print "WARNING: coef count didn't match feature count. multi-class classification was not supported"
# Calculate prediction and Save testing dataset
bt_coef_arr = sc.broadcast(coef_arr)
bt_intercept = sc.broadcast(intercept)
bt_jfeat_coef_dict = sc.broadcast(jfeat_coef_dict)
### Evaluating the model on testing dataset: label, predict label, score, feature list
print "INFO: intercept=", intercept
print "INFO: coef_arr len=", len(coef_arr), type(coef_arr)
print "INFO: jfeat_coef_dict len=", len(
jfeat_coef_dict) #, jfeat_coef_dict
# get prediction of testing dataset : (tlabel, plabel, score, libsvm, raw feat str, hash) ==============================
if len(coef_arr) == feature_count:
testing_pred_rdd = testing_rdd.map(lambda p: (
p[0].label \
,model_classification.predict(p[0].features) \
,zip_feature_util.calculate_hypothesis(p[0].features, bt_coef_arr.value, bt_intercept.value, model_name) \
,p[0].features \
,p[1] \
) ).cache()
else: # for multi-class, no prediction score; TBD for better solution: how to display multiple weights for each class
testing_pred_rdd = testing_rdd.map(lambda p: (
p[0].label \
,model_classification.predict(p[0].features) \
,"-" \
,p[0].features \
,p[1] \
) ).cache()
''',p[0].features.dot(bt_coef_arr.value)+bt_intercept.value \
# Save testing dataset for analysis
libsvm_testing_output = hdfs_feat_dir + "libsvm_testing_output_"+row_id_str
print "INFO: libsvm_testing_output=", libsvm_testing_output
try:
hdfs.rmr(libsvm_testing_output)
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(e.errno, e.strerror)
except:
print "WARNING: Unexpected error at libsvm_testing_output file clean up:", sys.exc_info()[0]
# save only false prediction?
#testing_pred_rdd.filter(lambda p: p[0] != p[1]).saveAsTextFile(libsvm_testing_output)
testing_pred_rdd.saveAsTextFile(libsvm_testing_output)
'''
#test_tmp=testing_pred_rdd.collect()
# save false prediction to local file
false_pred_fname = os.path.join(local_out_dir,
row_id_str + "_false_pred.json")
print "INFO: false_pred_fname=", false_pred_fname
false_pred_data=testing_pred_rdd.filter(lambda p: p[0] != p[1])\
.map(lambda p: (p[0],p[1],p[2] \
,zip_feature_util.get_dict_coef_raw4feat(zip_feature_util.sparseVector2dict(p[3]), bt_jfeat_coef_dict.value)
,p[4] ) ) \
.collect()
print "INFO: false predicted count=", len(false_pred_data)
false_pred_arr = []
with open(false_pred_fname, "w") as fp:
for sp in false_pred_data:
jsp = {
"tlabel": sp[0],
"plabel": sp[1],
"score": sp[2],
"feat": sp[3],
"hash": sp[4]
}
#print "jsp=",jsp
false_pred_arr.append(jsp)
fp.write(json.dumps(false_pred_arr))
# save prediction results, format: label, prediction, hash
pred_ofname = os.path.join(local_out_dir,
row_id_str + "_pred_output.pkl")
print "INFO: pred_ofname=", pred_ofname
pred_out_arr = testing_pred_rdd.map(lambda p:
(p[0], p[1], p[4])).collect()
ml_util.ml_pickle_save(pred_out_arr, pred_ofname)
'''
one_item= testing_pred_rdd.first()
print "one_item=",one_item
sparse_arr=one_item[3]
dict_feat=zip_feature_util.sparseVector2dict(sparse_arr)
print "len=",len(dict_feat),"dict_feat=",dict_feat
dict_weit=zip_feature_util.add_coef2dict(coef_arr,dict_feat)
print "len=",len(dict_weit),"dict_weit=",dict_weit
'''
# Calculate Accuracy. labelsAndPreds = (true_label,predict_label)
labelsAndPreds = testing_pred_rdd.map(lambda p: (p[0], p[1]))
labelsAndPreds.cache()
testing_sample_number = testing_rdd.count()
testErr = labelsAndPreds.filter(lambda (v, p): v != p).count() / float(
testing_sample_number)
accuracy = 1 - testErr
print "INFO: Accuracy = ", accuracy
### Save model
#save_dir = config.get('app', 'HADOOP_MASTER')+'/user/hadoop/yigai/row_6/'
#save_dir = config.get('app', 'HADOOP_MASTER')+config.get('app', 'HDFS_MODEL_DIR')+'/'+row_id_str
save_dir = os.path.join(config.get('app', 'HADOOP_MASTER'),
config.get('app', 'HDFS_MODEL_DIR'), row_id_str)
try:
hdfs.ls(save_dir)
#print "find hdfs folder"
hdfs.rmr(save_dir)
#print "all files removed"
except IOError as e:
print "WARNING: I/O error({0}): {1}".format(
e.errno, e.strerror), ". At HDFS=", save_dir
except:
print "WARNING: Unexpected error:", sys.exc_info(
)[0], ". At HDFS=", save_dir
model_classification.save(sc, save_dir)
###load model if needed
#sameModel = SVMModel.load(sc, save_dir)
t1 = time()
print 'INFO: training run time: %f' % (t1 - t0)
t0 = t1
###############################################
###########plot prediction result figure ==================================================== ===============
###############################################
labels = labelsAndPreds.collect()
true_label_list = [x for x, _ in labels]
pred_label_list = [x for _, x in labels]
pred_fname = os.path.join(local_out_dir, row_id_str + "_1" + ".png")
true_fname = os.path.join(local_out_dir, row_id_str + "_2" + ".png")
pred_xlabel = 'Prediction (Single Run)'
true_xlabel = 'True Labels (Single Run)'
test_cnt_dic = ml_util.ml_plot_predict_figures(
pred_label_list, true_label_list, labels_list, label_dic,
testing_sample_count, pred_xlabel, pred_fname, true_xlabel, true_fname)
print "INFO: figure files: ", pred_fname, true_fname
#print "INFO: Number of samples in each label is=", test_cnt_dic
roc_auc = None
perf_measures = None
dataset_info = {
"training_fraction": training_fraction,
"class_count": class_num,
"dataset_count": sample_count
}
#############################################################
###################for 2 class only (plot ROC curve) ==================================================== ===============
#############################################################
if len(labels_list) == 2:
do_ROC = True
reverse_label_dic = dict((v, k) for k, v in label_dic.items())
if 'clean' in reverse_label_dic:
flag_clean = reverse_label_dic['clean']
elif 'benign' in reverse_label_dic:
flag_clean = reverse_label_dic['benign']
elif '0' in reverse_label_dic:
flag_clean = 0
else:
print "INFO: No ROC curve generated: 'clean','benign' or '0' must be a label for indicating negative class!"
do_ROC = False
# build data file for score graph
score_graph_fname = os.path.join(local_out_dir,
row_id_str + "_score_graph.json")
print "INFO: score_graph_fname=", score_graph_fname
# build score_arr_0, score_arr_1
# format: tlabel, plabel, score, libsvm, raw feat str, hash
graph_arr = testing_pred_rdd.map(lambda p:
(int(p[0]), float(p[2]))).collect()
score_arr_0 = []
score_arr_1 = []
max_score = 0
min_score = 0
for p in graph_arr:
if p[0] == 0:
score_arr_0.append(p[1])
else:
score_arr_1.append(p[1])
# save max,min score
if p[1] > max_score:
max_score = p[1]
elif p[1] < min_score:
min_score = p[1]
ml_build_pred_score_graph(score_arr_0, score_arr_1, model_name,
score_graph_fname, max_score, min_score)
if do_ROC:
perf_measures = ml_util.calculate_fscore(true_label_list,
pred_label_list)
print "RESULT: perf_measures=", perf_measures
'''
# calculate fscore ==========
tp = labelsAndPreds.filter(lambda (v, p): v == 1 and p==1 ).count()
fp = labelsAndPreds.filter(lambda (v, p): v == 0 and p==1 ).count()
fn = labelsAndPreds.filter(lambda (v, p): v == 1 and p==0 ).count()
tn = labelsAndPreds.filter(lambda (v, p): v == 0 and p==0 ).count()
print "RESULT: tp=",tp,",fp=",fp,",fn=",fn,",tn=",tn
precision=float(tp)/(tp+fp)
recall=float(tp)/(tp+fn)
print "RESULT: precision=",precision,",recall=",recall
acc=(tp+tn)/(float(testing_sample_number))
fscore=2*((precision*recall)/(precision+recall))
print "RESULT: fscore=",fscore,",acc=",acc
'''
model_classification.clearThreshold()
scoreAndLabels = testing_rdd.map(lambda p: (
model_classification.predict(p[0].features), int(p[0].label)))
#metrics = BinaryClassificationMetrics(scoreAndLabels)
#areROC = metrics.areaUnderROC
#print areROC
scoreAndLabels_list = scoreAndLabels.collect()
if flag_clean == 0:
scores = [x for x, _ in scoreAndLabels_list]
s_labels = [x for _, x in scoreAndLabels_list]
testing_N = test_cnt_dic[0]
testing_P = test_cnt_dic[1]
else:
scores = [-x for x, _ in scoreAndLabels_list]
s_labels = [1 - x for _, x in scoreAndLabels_list]
testing_N = test_cnt_dic[1]
testing_P = test_cnt_dic[0]
# create ROC data file ======== ====
roc_auc = ml_create_roc_files(row_id_str, scores, s_labels,
testing_N, testing_P, local_out_dir,
row_id_str)
#, local_out_dir, file_name_given)
perf_measures["roc_auc"] = roc_auc
# only update db for web request ==================================================== ===============
if fromweb == "1":
#print "database update"
str_sql="UPDATE atdml_document set "+"accuracy = '"+str(accuracy*100)+"%" \
+"', status = 'learned', processed_date ='"+str(datetime.datetime.now()) \
+"', perf_measures='"+json.dumps(perf_measures) \
+"', dataset_info='"+json.dumps(dataset_info) \
+"' where id="+row_id_str
ret = exec_sqlite.exec_sql(str_sql)
print "INFO: Data update done! ret=", str(ret)
else:
print "INFO: accuracy = '" + str(accuracy * 100) + "%"
print 'INFO: Finished!'
return 0
def feat_importance_firm(row_id_str, ds_id, hdfs_feat_dir, local_score_file,
sp_master, spark_rdd_compress,
spark_driver_maxResultSize, sp_exe_memory,
sp_core_max, zipout_dir, zipcode_dir, zip_file_name,
mongo_tuples, training_fraction, jobname, uploadtype,
description_file):
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_util.ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
t0 = time()
# get feature seq mapping from mongo
if uploadtype == "MD5 List IN-dynamic":
### connect to database to get the column list which contains all column number of the corresponding feature
key = "dict_dynamic"
jstr_filter = '{"rid":' + row_id_str + ',"key":"' + key + '"}'
jstr_proj = '{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter = '{"rid":' + ds_id + ',"key":"' + key + '"}'
doc = query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_list = doc['value']
dic_all_columns = {}
max_feature = 0
# reverse dict{hashes:sequence number} ======
for i in range(0, len(dic_list)):
for key in dic_list[i]:
dic_all_columns[eval(dic_list[i][key])] = key
if eval(dic_list[i][key]) > max_feature:
max_feature = eval(dic_list[i][key])
print "INFO: max_feature=", max_feature
#print "dic_all_columns=",dic_all_columns # fid:numb,numb
dirFile_loc = os.path.join(hdfs_feat_dir, "metadata")
dirFolders = sc.textFile(dirFile_loc)
hash_Folders = dirFolders.collect()
#print "INFO: dirFile_loc=",dirFile_loc,", hash_Folders=",hash_Folders
folder_list = [x.encode('UTF8') for x in hash_Folders]
print "INFO: hdfs folder_list=", folder_list #['dirty/', 'clean/']
features_training = []
labels_training = []
names_training = []
row_training = []
col_training = []
max_feat_training = 0
row_num_training = 0
features_testing = []
labels_testing = []
names_testing = []
row_testing = []
col_testing = []
max_feat_testing = 0
row_num_testing = 0
# loop through hdfs folders; TBD
for folder in folder_list:
print "INFO: folder=", folder
label = folder_list.index(folder) + 1
print 'INFO: label=', label
logFile_name = os.path.join(hdfs_feat_dir, folder, mtx_name_list)
#print "logFile_name=",logFile_name
logFile_data = os.path.join(hdfs_feat_dir, folder, mtx_libsvm)
#print "logFile_data=",logFile_data
logNames = sc.textFile(logFile_name).cache()
logData = sc.textFile(logFile_data).cache()
names = logNames.collect()
data = logData.collect()
name_l = [x.encode('UTF8') for x in names]
feature_l = [x.encode('UTF8') for x in data]
name_list = [names.strip() for names in name_l]
feature_list = [features.strip() for features in feature_l]
##########data seperation######
id_perm = data_seperation_random(name_list)
num_names = len(name_list)
print 'INFO: num of samples=', num_names
num_train = int(training_portion * num_names)
print 'INFO: num_train = ', num_train
########generate training data#########
i = 0
#print "INFO: generate training data"
#print "INFO: len(id_perm)=",len(id_perm)
while i < num_train:
#print i, id_perm[i]
features = feature_list[id_perm[i]]
features = features.strip()
feature_array = features.split(' ')
labels_training.append(label)
length = len(feature_array)
j = 0
while j < length:
feature = feature_array[j]
feat, value = feature.split(':', 2)
row_training.append(i + row_num_training)
col_training.append(int(feat) - 1)
features_training.append(int(value))
max_feat_training = max(max_feat_training, int(feat))
j = j + 1
i = i + 1
row_num_training = row_num_training + num_train
i = num_train
########generate testing data#########
while i < num_names:
####for generating testing data folder####
test_file_name = name_list[id_perm[i]]
features = feature_list[id_perm[i]]
features = features.strip()
feature_array = features.split(' ')
labels_testing.append(label)
length = len(feature_array)
j = 0
while j < length:
feature = feature_array[j]
feat, value = feature.split(':', 2)
row_testing.append(i - num_train + row_num_testing)
col_testing.append(int(feat) - 1)
features_testing.append(int(value))
max_feat_testing = max(max_feat_testing, int(feat))
j = j + 1
i = i + 1
row_num_testing = row_num_testing + (num_names - num_train)
# end for loop here ========================
col_num = max(max_feat_training, max_feat_testing)
if max_feat_training < col_num:
for i in range(0, row_num_training):
for j in range(max_feat_training, col_num):
features_training.append(0)
row_training.append(i)
col_training.append(j)
elif max_feat_testing < col_num:
for i in range(0, row_num_testing):
for j in range(max_feat_testing, col_num):
features_testing.append(0)
row_testing.append(i)
col_testing.append(j)
features_training = array(features_training)
row_training = array(row_training)
col_training = array(col_training)
#print "row_training:", row_training
#print "INFO: col_training:", col_training
len_col = len(col_training)
print "INFO: col_num:", col_num
labels_training = array(labels_training)
features_testing = array(features_testing)
row_testing = array(row_testing)
col_testing = array(col_testing)
labels_testing = array(labels_testing)
sparse_mtx = csc_matrix((features_training, (row_training, col_training)),
shape=(row_num_training, col_num))
#print "sparse_mtx.todense(), sparse_mtx.shape=",sparse_mtx.todense(), sparse_mtx.shape
sparse_test = csc_matrix((features_testing, (row_testing, col_testing)),
shape=(row_num_testing, col_num))
#print " sparse_test.todense(), sparse_test.shape=",sparse_test.todense(), sparse_test.shape
clf = svm.LinearSVC()
#clf = svm.SVC(C=0.1, kernel='rbf', degree=3, gamma=0.05, coef0=0.0, shrinking=True, probability=False, tol=0.001, cache_size=200, class_weight=None, verbose=False, max_iter=-1, random_state=None)
#clf = svm.NuSVC(nu=0.3, kernel='rbf', degree=3, gamma=0.05, coef0=0.0, shrinking=True, probability=False, tol=0.001, cache_size=200, verbose=False, max_iter=-1, random_state=None)
#print "labels_training=",labels_training
#print "sparse_mtx=",sparse_mtx
clf.fit(sparse_mtx, labels_training)
#print "INFO: model:intercept=",clf.intercept_
#print "INFO: model:coef=",clf.coef_
labels_pred = clf.predict(sparse_test)
#print "labels_pred:", labels_pred
accuracy = clf.score(sparse_test, labels_testing)
#print "INFO: data folder=", hdfs_feat_dir
print "INFO: accuracy=", accuracy
#####################################################################
##################calculate feature importance with predication labels#######################
#####################################################################
AA = sparse_mtx.todense()
BB = sparse_test.todense()
labels_train_pred = clf.predict(sparse_mtx)
labels_test_pred = labels_pred
#print "###################################################################################"
print "INFO: ======= Calculate feature importance with predication labels =================="
#print "###################################################################################"
dic_importance_label = {}
for j in range(0, col_num): ###for all features in the loop
##############################
#print "====new way with sparse matrix========="
curr_col_train = sparse_mtx.getcol(j)
sum_col = curr_col_train.sum(0)
positive_feature_number = int(sum_col.tolist()[0][0])
labels_value = 3 - labels_train_pred
dot_product = csr_matrix(np.array(labels_value)).dot(curr_col_train)
sum_product = dot_product.sum(1)
labels_positive_sum = int(sum_product.tolist()[0][0])
sum_label_values = sum(labels_value)
labels_negitive_sum = sum_label_values - labels_positive_sum
##############################
#print "====new way with sparse matrix========="
curr_col_test = sparse_test.getcol(j)
sum_col = curr_col_test.sum(0)
positive_feature_number = positive_feature_number + int(
sum_col.tolist()[0][0])
labels_value = 3 - labels_test_pred
dot_product = csr_matrix(np.array(labels_value)).dot(curr_col_test)
sum_product = dot_product.sum(1)
labels_positive_sum = labels_positive_sum + int(
sum_product.tolist()[0][0])
sum_label_values = sum(labels_value)
labels_negitive_sum = labels_negitive_sum + sum_label_values - int(
sum_product.tolist()[0][0])
n_total = row_num_training + row_num_testing
negitive_feature_number = n_total - positive_feature_number
if positive_feature_number == 0:
#print "feature ", j+1, "all 0s!"
dic_importance_label[j + 1] = -100
elif negitive_feature_number == 0:
#print "feature ", j+1, "all 1s!"
dic_importance_label[j + 1] = -200
else:
q_positive = float(labels_positive_sum) / positive_feature_number
q_negitive = float(labels_negitive_sum) / negitive_feature_number
Q = (q_positive - q_negitive) * sqrt(
float(q_positive) * q_negitive / float(n_total) /
float(n_total))
dic_importance_label[j + 1] = Q
sorted_importance = sorted(dic_importance_label.items(),
key=operator.itemgetter(1),
reverse=True)
print "INFO: ======= Feature Importance(FIRM score) ================"
if os.path.exists(local_score_file):
try:
os.remove(local_score_file)
except OSError, e:
print("ERROR: %s - %s." % (e.local_score_file, e.strerror))
def feat_importance_firm(row_id_str, ds_id, hdfs_feat_dir, local_score_file,
sp_master, spark_rdd_compress,
spark_driver_maxResultSize, sp_exe_memory,
sp_core_max, zipout_dir, zipcode_dir, zip_file_name,
mongo_tuples, training_fraction, jobname, uploadtype):
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_util.ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
'''
SparkContext.setSystemProperty('spark.rdd.compress', config.get('spark', 'spark_rdd_compress'))
SparkContext.setSystemProperty('spark.driver.maxResultSize', config.get('spark', 'spark_driver_maxResultSize'))
SparkContext.setSystemProperty('spark.executor.memory', args.exe_memory)
SparkContext.setSystemProperty('spark.cores.max', args.core_max)
sc = SparkContext(args.sp_master, 'feature_importance_FRIM:'+str(args.row_id))
'''
t0 = time()
# get folder list (labels) from hdfs data_out/<id>/metadata ==============
dirFile_loc = os.path.join(hdfs_feat_dir, "metadata")
dirFolders = sc.textFile(dirFile_loc)
hash_Folders = dirFolders.collect()
print "INFO: dirFile_loc=", dirFile_loc, ", hash_Folders=", hash_Folders
folder_list = [x.encode('UTF8') for x in hash_Folders]
print "INFO: folder_list=", folder_list #['dirty/', 'clean/']
# get feature seq : ngram hash mapping ==================================
key = "dic_seq_hashes" #{"123":"136,345"}
jstr_filter = '{"rid":' + row_id_str + ',"key":"' + key + '"}'
jstr_proj = '{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter = '{"rid":' + ds_id + ',"key":"' + key + '"}'
doc = query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_list = doc['value']
dic_all_columns = dic_list
feature_count = len(dic_list)
#print "INFO: feature_count=",feature_count
#print "dic_list=",dic_list #{u'123,345':u'136'}
#print "dic_all_columns=",dic_all_columns # {1: u'8215,8216'}
# end
# get {hash : raw string} mapping ==================================
key = "dic_hash_str" #{"123":"openFile"}
jstr_filter = '{"rid":' + row_id_str + ',"key":"' + key + '"}'
jstr_proj = '{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter = '{"rid":' + ds_id + ',"key":"' + key + '"}'
doc = query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_hash_str = doc['value']
'''
# get folder list (labels) from hdfs data_out/<id>/libsvm ==============
libsvm_loc = os.path.join(hdfs_feat_dir , "libsvm_data")
print "INFO: libsvm_loc=", libsvm_loc
samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_loc)
'''
# filename for featured data
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
# load feature count file
#feat_count_file=libsvm_data_file+"_feat_count"
#feature_count=zip_feature_util.get_feature_count(sc,feat_count_file)
print "INFO: feature_count=", feature_count
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
# load sample RDD from text file
# also exclude selected features in sample ================ =====
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(
sc, libsvm_data_file, feature_count, excluded_feat_cslist=None)
labels_and_features_rdd = samples_rdd.map(lambda p:
(p[0].label, p[0].features))
all_data = labels_and_features_rdd.collect()
features_list = [x.toArray() for _, x in all_data]
labels_list_all = [x for x, _ in all_data]
labels_list_all = np.array(labels_list_all)
features_array = np.array(features_list)
### generate sparse matrix (csr) for all samples
features_sparse_mtx = csr_matrix(features_array)
### randomly split the samples into training and testing data
sparse_mtx, sparse_test, labels_training, labels_testing = \
cross_validation.train_test_split(features_sparse_mtx, labels_list_all, test_size=(1-training_fraction))
#print "INFO: sparse_mtx.shape=",sparse_mtx.shape
#print "INFO: sparse_test.shape=",sparse_test.shape
row_num_training = (sparse_mtx.shape)[0]
row_num_testing = (sparse_test.shape)[0]
# why use LinearSVC ?
clf = svm.LinearSVC()
#clf = svm.SVC(C=0.1, kernel='rbf', degree=3, gamma=0.05, coef0=0.0, shrinking=True, probability=False, tol=0.001, cache_size=200, class_weight=None, verbose=False, max_iter=-1, random_state=None)
#clf = svm.NuSVC(nu=0.3, kernel='rbf', degree=3, gamma=0.05, coef0=0.0, shrinking=True, probability=False, tol=0.001, cache_size=200, verbose=False, max_iter=-1, random_state=None)
#print "labels_training=",labels_training
#print "sparse_mtx=",sparse_mtx
clf.fit(sparse_mtx, labels_training)
#print "**model:intercept***"
#print clf.intercept_
#print "**model:coef***"
#print clf.coef_
col_num = len(clf.coef_[0]) # for n_classes==2
print "INFO: col_num=", col_num
labels_pred = clf.predict(sparse_test)
#print "labels_pred:", labels_pred
accuracy = clf.score(sparse_test, labels_testing)
print "INFO: data folder:", hdfs_feat_dir
print "INFO: accuracy: ", accuracy
#####################################################################
##################calculate feature importance with predication labels#######################
#####################################################################
AA = sparse_mtx.todense()
BB = sparse_test.todense()
labels_train_pred = clf.predict(sparse_mtx)
labels_test_pred = labels_pred
print "INFO: ###################################################################################"
print "INFO: ############calculate feature importance with predication labels###################"
print "INFO: ###################################################################################"
dic_importance_label = {}
for j in range(0, col_num): ###for all features in the loop
##############################
#print "====new way with sparse matrix========="
curr_col_train = sparse_mtx.getcol(j)
sum_col = curr_col_train.sum(0)
positive_feature_number = int(sum_col.tolist()[0][0])
labels_value = 3 - labels_train_pred
dot_product = csr_matrix(np.array(labels_value)).dot(curr_col_train)
sum_product = dot_product.sum(1)
labels_positive_sum = int(sum_product.tolist()[0][0])
sum_label_values = sum(labels_value)
labels_negitive_sum = sum_label_values - labels_positive_sum
##############################
#print "====new way with sparse matrix========="
curr_col_test = sparse_test.getcol(j)
sum_col = curr_col_test.sum(0)
positive_feature_number = positive_feature_number + int(
sum_col.tolist()[0][0])
labels_value = 3 - labels_test_pred
dot_product = csr_matrix(np.array(labels_value)).dot(curr_col_test)
sum_product = dot_product.sum(1)
labels_positive_sum = labels_positive_sum + int(
sum_product.tolist()[0][0])
sum_label_values = sum(labels_value)
labels_negitive_sum = labels_negitive_sum + sum_label_values - int(
sum_product.tolist()[0][0])
n_total = row_num_training + row_num_testing
negitive_feature_number = n_total - positive_feature_number
if positive_feature_number == 0:
#print "feature ", j+1, "all 0s!"
dic_importance_label[j + 1] = -100
elif negitive_feature_number == 0:
#print "feature ", j+1, "all 1s!"
dic_importance_label[j + 1] = -200
else:
q_positive = float(labels_positive_sum) / positive_feature_number
q_negitive = float(labels_negitive_sum) / negitive_feature_number
Q = (q_positive - q_negitive) * sqrt(
float(q_positive) * q_negitive / float(n_total) /
float(n_total))
dic_importance_label[j + 1] = Q
sorted_importance = sorted(dic_importance_label.items(),
key=operator.itemgetter(1),
reverse=True)
print "INFO: =======Feature Importance(FIRM score)================"
if os.path.exists(local_score_file):
try:
os.remove(local_score_file)
except OSError, e:
print("ERROR: %s - %s." % (e.local_score_file, e.strerror))
def preprocess(
row_id_str,
ds_id,
hdfs_feat_dir,
local_out_dir,
ml_opts_jstr #, excluded_feat_cslist
,
sp_master,
spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory,
sp_core_max,
zipout_dir,
zipcode_dir,
zip_file_name,
data_fname
#, mongo_tuples, labelnameflag, fromweb
,
jobname,
dnn_data_suffix=config.get("machine_learning", "dnn_data_suffix"),
dnn_label_suffix=config.get("machine_learning", "dnn_label_suffix"),
dnn_info_suffix=config.get("machine_learning", "dnn_info_suffix")):
### generate data folder and out folder, clean up if needed
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
# create zip files for Spark workers ================= ================
zip_file_path = ml_util.ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path, ",ml_opts_jstr=", ml_opts_jstr
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
t0 = time()
# Check option
ml_opts = {}
try:
ml_opts = json.loads(ml_opts_jstr)
except:
print "ERROR: string ml_opts is invalid!"
return -1
learning_algorithm = ml_opts["learning_algorithm"]
#return
# create 3 arrays
feat_list = []
label_list = []
info_list = []
feature_count = None
sample_count = None
if learning_algorithm == "cnn": #================================================== ==========
# libsvm_data for featured data, not use "dnn_data" here
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
# load sample RDD from libsvm file
# output format: [([features], label, info)] , feature_count, feat_max, feat_min
all_list, feature_count, feat_max, feat_min =\
zip_feature_util.get_sample_as_arr(sc, libsvm_data_file, None, None)
c = 0
# convert to 3 list ========================= ==========
for i in all_list:
#print type(i[0]),type(i[1]), type(i[2])
if c % 10000 == 0:
print ".",
sys.stdout.flush()
c = c + 1
# convert to float32 np array and normalize by max value (0-1.0)
feat_list.append(np.array(i[0], dtype=np.float32) / feat_max)
label_list.append(i[1])
info_list.append(i[2])
print "INFO: feat_list t=", type(feat_list), type(feat_list[0]), type(
feat_list[0][0])
elif learning_algorithm == "lstm": #================================================== ==========
# filename for featured data
dnn_data_file = os.path.join(hdfs_feat_dir, data_fname)
print "INFO: dnn_data_file=", dnn_data_file
# output format: [([features], label, info)] , chucked_sample_count
all_list, chucked_sample_count=\
zip_feature_util.get_sample_as_chunk(sc, dnn_data_file, None, None)
c = 0
# convert to 3 list ========================= ==========
for i in all_list:
#print type(i[0]),type(i[1]), type(i[2])
if c % 10000 == 0:
print ".",
sys.stdout.flush()
c = c + 1
# convert to float32 np array and normalize by max value (0-1.0)
feat_list.append(np.array(i[0], dtype=np.int32))
label_list.append(i[1])
info_list.append(i[2])
print "INFO: feat_list t=", type(feat_list), type(
feat_list[0]), feat_list[0][0]
print "INFO: label_list t=", type(label_list), label_list[0]
print "INFO: info_list t=", type(info_list), info_list[0]
if not all_list is None and len(all_list) > 0:
sample_count = len(all_list)
else:
sample_count = None
print "INFO: sample_count=", sample_count
tgt_prefix = row_id_str
if (row_id_str != ds_id):
tgt_prefix = ds_id
# save data file ========================= ==========
data_fname = os.path.join(local_out_dir, tgt_prefix + dnn_data_suffix)
print "INFO: data_fname=", data_fname
# reshape for image
#if not feature_count is None and feature_count >0 and not sample_count is None and sample_count > 0:
# nparr_feat=np.asarray(feat_list,dtype=np.float32).reshape(sample_count,feature_count)
#else: # lstm didn't need to reshape
nparr_feat = np.asarray(feat_list, dtype=np.int32)
print "INFO: nparr_feat s=", nparr_feat.shape, nparr_feat[0].shape, type(
nparr_feat[0][0]), nparr_feat[0][0]
with gzip.open(data_fname, "wb") as fp:
#pickle.dump(nparr_feat,fp, 2) # may overflow in pickle; limited to 2B elements; use numpy.save()
np.save(fp, nparr_feat, allow_pickle=False) #62.9M for (465724, 2967)
# save label file ========================= ==========
lbl_fname = os.path.join(local_out_dir, tgt_prefix + dnn_label_suffix)
print "INFO: lbl_fname=", lbl_fname
# convert to numpy array and int32 to save space
nparr_lbl = np.asarray(label_list, dtype=np.int32)
#print "label=",nparr_lbl[0]
with gzip.open(lbl_fname, "wb") as fp:
#pickle.dump(nparr_lbl,fp, 2)
np.save(fp, nparr_lbl, allow_pickle=False)
# save info file ========================= ==========
info_fname = os.path.join(local_out_dir, tgt_prefix + dnn_info_suffix)
print "INFO: info_fname=", info_fname
nparr_info = np.asarray(info_list)
with gzip.open(info_fname, "wb") as fp:
#pickle.dump(nparr_info,fp,2)
np.save(fp, nparr_info, allow_pickle=False)
t1 = time()
print 'INFO: running time: %f' % (t1 - t0)
print 'INFO: Finished!'
return 0
def pca(row_id_str, ds_id, hdfs_feat_dir, local_out_dir, sp_master,
spark_rdd_compress, spark_driver_maxResultSize, sp_exe_memory,
sp_core_max, zipout_dir, zipcode_dir, zip_file_name, mongo_tuples,
fromweb, pca_jstr, jobname, model_data_folder):
# create zip files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# init Spark context ====
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
'''
SparkContext.setSystemProperty('spark.rdd.compress', config.get('spark', 'spark_rdd_compress'))
SparkContext.setSystemProperty('spark.driver.maxResultSize', config.get('spark', 'spark_driver_maxResultSize'))
SparkContext.setSystemProperty('spark.executor.memory', args.exe_memory)
SparkContext.setSystemProperty('spark.cores.max', args.core_max)
sc = SparkContext(args.sp_master, 'pca_sklearn:'+str(args.row_id))
'''
pca_param = json.loads(pca_jstr)
if "k" in pca_param:
k = pca_param["k"]
else:
k = None
if "threshold" in pca_param:
threshold = pca_param["threshold"]
else:
threshold = None
if "lib" in pca_param:
lib = pca_param["lib"]
else:
lib = 'sklearn'
#if "recreate" in pca_param:
# recreate=pca_param["recreate"]
#else:
# recreate='0'
ret = -1
# start here =================================================================== ===============
t0 = time()
# source libsvm filename
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
# load feature count file
feat_count_file = libsvm_data_file + "_feat_count"
feature_count = zip_feature_util.get_feature_count(sc, feat_count_file)
print "INFO: feature_count=", feature_count
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
# load sample RDD from text file
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(
sc, libsvm_data_file, feature_count, '')
# convert labeled point to tuple (label,features)
#labels_and_features_rdd = samples_rdd.map(lambda p: (p.label, p.features))
all_data = samples_rdd.collect()
sample_count = len(all_data)
# 2-D array
features_list = [x.features.toArray() for x, _ in all_data]
# label array
labels_list_all = [x.label for x, _ in all_data]
# hash array
hash_list_all = [x for _, x in all_data]
# convert to int to speed up
features_array = np.array(features_list, dtype=np.int32)
true_label_array = np.array(labels_list_all, dtype=np.int8)
hash_list_all = np.array(hash_list_all)
# PCA here, TBD for removal
print "INFO: features_array.shape=", features_array.shape
print "INFO: true_label_array.shape=", true_label_array.shape
print "INFO: hash_list_all.shape=", hash_list_all.shape
print "INFO: Doing PCA... threshold=", threshold, ",k=", k
(features_array_reduced, k,
pca) = sklearn_PCA_transform(features_array, threshold, k)
#print "features_array_reduced t=", type(features_array_reduced), ",features_array_reduced=", features_array_reduced
print "INFO: features_array_reduced.shape", features_array_reduced.shape, ",k=", k
pca_fname3 = ""
if not threshold is None:
pca_fname = os.path.join(
model_data_folder, row_id_str + '_pca_' + str(threshold) + '.pkl')
pca_fname3 = os.path.join(
model_data_folder, row_id_str + '_pca3_' + str(threshold) + '.pkl')
else:
pca_fname = os.path.join(model_data_folder,
row_id_str + '_pca_' + str(k) + '.pkl')
#pc_3=features_array_reduced[:,0:3]
#print "type=",type(pc_3),"shape=",pc_3.shape,"pca_fname3=",pca_fname3
#np.save(pca_fname3,pc_3)
if not os.path.exists(model_data_folder):
os.makedirs(model_data_folder)
if os.path.exists(pca_fname): # remove file if exists
try:
for fl in glob.glob(pca_fname + "*"):
os.remove(fl)
except OSError, e:
print("Error: %s - %s." % (e.pca_fname, e.strerror))
def exclude_sample(hdfs_src_dir,
fname_list,
hdfs_out_dir,
hdfs_excl_dir,
hdfs_excl_fname_list,
sp_master,
spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory,
sp_core_max,
jobname,
delimitor="_"):
# clean up output hdfs folder
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname)
t0 = time()
# get exclusion data
ex_files = os.path.join(hdfs_excl_dir, hdfs_excl_fname_list)
if ',' in hdfs_excl_fname_list:
ex_files = ""
comma = ""
for fn in hdfs_excl_fname_list.split(','):
ex_files = ex_files + comma + os.path.join(hdfs_excl_dir, fn)
comma = ","
print "INFO: mc only files=", ex_files
ex_rdd = sc.textFile(ex_files).map(lambda x: (x, x)).distinct().cache()
# get source files by pattern: the 1st part of filename
if fname_list == "_1st_part_":
f_list = hdfs.ls(hdfs_src_dir)
f_dict = {}
for i in f_list:
bname = os.path.basename(i)
pattern = bname
if bname.index('_') > 0:
pattern = bname[:bname.index('_') + 1] + '*.gz'
dname = os.path.dirname(i)
pname = os.path.join(dname, pattern)
#
if not pname in f_dict:
#print "p:",pname
f_dict[pname] = 1
# create list string
fname_list = ','.join(f_dict)
# for compression
codec = "org.apache.hadoop.io.compress.GzipCodec"
# for each source files, exclude data and save to output
for fn in sorted(fname_list.split(',')):
if len(fn) == 0:
continue
ofn = None
if not 'hdfs://' in fn:
src_file = os.path.join(hdfs_src_dir, fn)
# assume output name is the 1st part of input fname
ofn = fn.partition(delimitor)[0]
out_file = os.path.join(hdfs_out_dir, ofn)
else: # list from "_1st_part_"
src_file = fn
bname = os.path.basename(fn)
ofn = bname.partition(delimitor)[0]
out_file = os.path.join(hdfs_out_dir, ofn)
print "INFO: out files=", out_file, "ofn=", ofn
# clean up existing file
ml_util.ml_clean_up_hdfs_file(out_file)
#load data & set key; may have .<familyname>; left join, filter non-matching; remove key & save
src_rdd = None
cnts = 0
label_list = None
try:
src_rdd = sc.textFile(src_file)
#label_before=src_rdd.map(lambda x: x[:7]).distinct().collect()
#print "INFO: label_before=",label_before
src_rdd=src_rdd.filter(lambda x: not x is None) \
.map(lambda x: (x.split('\t'),x)) \
.filter(lambda x: len(x[0])>1) \
.cache()
#label_list=src_rdd.map(lambda x:x[0][0]).distinct().collect()
#print "INFO: label_list=",label_list
src_rdd = src_rdd.map(lambda x:
(x[0][1].split('.')[0], x[1])).cache()
# .map(lambda x: (x.split('\t')[1].split('.')[0],x)).cache()
cnts = src_rdd.count()
except:
print "WARNING:", sys.exc_info()[0]
#print "INFO: input ["+ofn+"] count=",cnt
if cnts > 0:
f_rdd=src_rdd \
.leftOuterJoin(ex_rdd) \
.filter(lambda x:x[1][1] is None) \
.map(lambda x:x[1][0]).cache()
cntf = f_rdd.count()
print "INFO: [" + ofn + "] remain ratio=", cntf, "/", cnts, "=", str(
cntf * 1.0 / cnts)
if cntf > 0:
f_rdd.saveAsTextFile(out_file, codec)
else:
print "WARNING: filter excluded all data for [" + ofn + "]"
else:
print "WARNING: [" + ofn + "] has no data"
#End for loop
t1 = time()
print 'INFO: running time: %f' % (t1 - t0)
print 'INFO: Finished!'
return 0
def mrun(row_id_str, ds_id, hdfs_feat_dir, local_out_dir, ml_opts_jstr,
excluded_feat_cslist, sp_master, spark_rdd_compress,
spark_driver_maxResultSize, sp_exe_memory, sp_core_max, zipout_dir,
zipcode_dir, zip_file_name, mongo_tuples, fromweb, training_fraction,
jobname, run_number, bin_number):
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# init Spark context ====
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
t0 = time()
# check if ml_opts.has_excluded_feat ==1 ===================================
has_excluded_feat = 0
if not ml_opts_jstr is None:
ml_opts = json.loads(ml_opts_jstr)
if "has_excluded_feat" in ml_opts:
has_excluded_feat = ml_opts["has_excluded_feat"]
# get excluded feature list from mongo ========== ===
if str(has_excluded_feat) == "1" and excluded_feat_cslist is None:
excluded_feat_cslist = ml_util.ml_get_excluded_feat(
row_id_str, mongo_tuples)
print "INFO: excluded_feat_cslist=", excluded_feat_cslist
### load libsvm file ###
#libsvm_data_file = data_folder + "libsvm_data"
# source libsvm filename
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file:", libsvm_data_file
# load feature count file
feat_count_file = libsvm_data_file + "_feat_count"
feature_count = zip_feature_util.get_feature_count(sc, feat_count_file)
print "INFO: feature_count=", feature_count
# load sample RDD from text file
# also exclude selected features in sample ================ =====
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(
sc, libsvm_data_file, feature_count, excluded_feat_cslist)
#samples_rdd = MLUtils.loadLibSVMFile(sc, libsvm_data_file)
#print samples_rdd.count()
# collect all data to local for processing ===============
all_data = samples_rdd.map(
lambda p: p[0]).collect() # keep LabeledPoint only
# 2-D array
features_list = [x.features.toArray() for x in all_data]
# label array
labels_list_all = [x.label for x in all_data]
# convert to np array
labels_list_all = array(labels_list_all)
features_array = np.array(features_list)
### generate sparse matrix (csr) for all samples
features_sparse_mtx = csr_matrix(features_array)
t1 = time()
print "INFO: labels_list_all=", labels_list_all
print 'INFO: data generating time: %f' % (t1 - t0)
label_set = set(labels_list_all)
class_num = len(label_set)
#class_num = len(labels_list)
if class_num > 2:
print "INFO: Number of classes =", class_num
###############################################
###########build learning model################
###############################################
### parse parameters and generate the model ###
(clf, model_name) = parse_para_and_get_model(ml_opts)
if model_name == "none":
return
t0 = time()
accuracy_array = np.zeros(run_number)
for rnd in range(0, run_number):
### randomly split the samples into training and testing data
X_train_sparse, X_test_sparse, labels_train, labels_test = \
cross_validation.train_test_split(features_sparse_mtx, labels_list_all, test_size=(1-training_fraction))
#####fit the model to training dataset ####
try:
clf.fit(X_train_sparse, labels_train)
except ValueError as v:
print "INFO: ValueError:", v
#raise v didn't work
return -1
### Evaluating the model on testing data
labels_pred = clf.predict(X_test_sparse)
accuracy = clf.score(X_test_sparse, labels_test)
accuracy_array[rnd] = accuracy
print "INFO: current round: ", rnd
print "INFO: Accuracy = ", accuracy
###############################################
#######plot distribution and variance##########
###############################################
plt.figure(1)
num_bins = bin_number ####10 is default
n, bins, patches = plt.hist(accuracy_array,
num_bins,
normed=1,
facecolor='green',
alpha=0.5)
ave = np.mean(accuracy_array)
print "INFO: num_bins=", num_bins
print "INFO: accuracy_array=", accuracy_array
print "INFO: Accuracy mean: ", ave
variance = np.std(accuracy_array)
print "INFO: Accuracy variance: ", variance
# add a 'best fit' line
y = mlab.normpdf(bins, ave, variance)
#print "INFO: y: ", y
plt.plot(bins, y, 'r--')
plt.title('Accuracy distribution of ' + str(run_number) + ' runs:')
plt.xlabel('Accuracy Values')
plt.ylabel('Probability / Accuracy bar width')
mrun_fname = os.path.join(local_out_dir,
row_id_str + "_var_" + str(run_number) + ".png")
plt.savefig(mrun_fname)
# create data for graph ====================
all_json = []
barp_arr = [] #n
disp_arr = [] #y
last_idx = 0
for idx, ht in enumerate(n): # n is bar height
#print "mrun bar=",idx, bins[idx], bins[idx+1],((bins[idx]+bins[idx+1])/2)
barp_arr.append([((bins[idx] + bins[idx + 1]) / 2.0),
n[idx]]) # mid point for x axis, may shift 100% bar
if not math.isnan(y[idx]):
disp_arr.append([bins[idx], y[idx]])
last_idx = idx
#print last_idx+1, bins[last_idx+1], y[last_idx+1]
if not math.isnan(y[last_idx + 1]):
disp_arr.append([bins[last_idx + 1], y[last_idx + 1]])
#print "barp_arr=", barp_arr
#print "disp_arr=", disp_arr
#bar
bar_json = {}
bar_json["values"] = barp_arr
bar_json["key"] = 'Mutil-Run Accuracy' #
bar_json["type"] = "bar" # light blue
bar_json["yAxis"] = 1
all_json.append(bar_json)
#distribution
if len(disp_arr) > 0:
dis_json = {}
dis_json["values"] = disp_arr
dis_json["key"] = 'Normal Distribution' #
dis_json["type"] = "line" # light blue
dis_json["yAxis"] = 1
all_json.append(dis_json)
mrun_jfile = os.path.join(local_out_dir, row_id_str + "_mrun.json")
#mrun_jfile = local_out_dir+row_id_str+"_mrun.json"
#print "INFO: all_json=",all_json
print "INFO: mrun_jfile=", mrun_jfile
if os.path.exists(mrun_jfile):
try:
os.remove(mrun_jfile)
except OSError, e:
print("ERROR: %s - %s." % (e.mrun_jfile, e.strerror))
def feat_importance_ip(row_id_str, ds_id, hdfs_feat_dir, local_score_file,
score_file_IT, sp_master, spark_rdd_compress,
spark_driver_maxResultSize, sp_exe_memory, sp_core_max,
zipout_dir, zipcode_dir, zip_file_name, mongo_tuples,
jobname, uploadtype):
# zip func in other files for Spark workers ================= ================
zip_file_path = ml_util.ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
# get_spark_context
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
'''
SparkContext.setSystemProperty('spark.rdd.compress', config.get('spark', 'spark_rdd_compress'))
SparkContext.setSystemProperty('spark.driver.maxResultSize', config.get('spark', 'spark_driver_maxResultSize'))
#SparkContext.setSystemProperty('spark.kryoserializer.buffer.mb', config.get('spark', 'spark_kryoserializer_buffer_mb'))
SparkContext.setSystemProperty('spark.executor.memory', args.exe_memory)
SparkContext.setSystemProperty('spark.cores.max', args.core_max)
sc = SparkContext(args.sp_master, 'feature_importance_2ways:'+str(args.row_id))
'''
t0 = time()
# get folder list (labels) from hdfs data_out/<id>/metadata ==============
dirFile_loc = os.path.join(hdfs_feat_dir, "metadata")
dirFolders = sc.textFile(dirFile_loc)
hash_Folders = dirFolders.collect()
print "INFO: dirFile_loc=", dirFile_loc, ", hash_Folders=", hash_Folders
folder_list = [x.encode('UTF8') for x in hash_Folders]
print "INFO: folder_list=", folder_list
# get feature seq : ngram hash mapping ==================================
key = "dic_seq_hashes" #{"123":"136,345"}
jstr_filter = '{"rid":' + row_id_str + ',"key":"' + key + '"}'
jstr_proj = '{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter = '{"rid":' + ds_id + ',"key":"' + key + '"}'
doc = query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_list = doc['value']
dic_all_columns = dic_list
feature_count = len(dic_list)
#print "INFO: feature_count=",feature_count
#print "dic_list=",dic_list #{u'123,345':u'136'}
#print "INFO: dic_all_columns=",dic_all_columns # {1: u'8215,8216'}
# end
# get hash : raw string mapping ==================================
key = "dic_hash_str" #{"123":"openFile"}
jstr_filter = '{"rid":' + row_id_str + ',"key":"' + key + '"}'
jstr_proj = '{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter = '{"rid":' + ds_id + ',"key":"' + key + '"}'
doc = query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_hash_str = doc['value']
'''
# get folder list (labels) from hdfs data_out/<id>/libsvm ==============
libsvm_loc = os.path.join(hdfs_feat_dir , "libsvm_data")
# based on label, divide RDD into arrays
f_rdd = sc.textFile(libsvm_loc).map(lambda x: libsvm2tuple_arr(x))
arr_libsvm=sorted(f_rdd.collect(), key=lambda x:x[0]) # sorted by label
'''
# filename for featured data
libsvm_data_file = os.path.join(hdfs_feat_dir, "libsvm_data")
print "INFO: libsvm_data_file=", libsvm_data_file
print "INFO: feature_count=",feature_count\
# get sample array from hdfs
arr_libsvm = zip_feature_util.get_sample_tuple_arr(sc, libsvm_data_file)
# sorted by label
arr_libsvm = sorted(arr_libsvm, key=lambda x: x[0])
# convert libsvm to features_list, row_list, col_list, sample count, col_num
lbl_flag = -1
row_num_training = 0
sparse_mtx_list = [] # for feat impor calculation
features_list = [] # for csc_matrix
row_list = [] # for csc_matrix
col_list = [] # for csc_matrix
sample_numbers = [] # for csc_matrix
feature_arr = None
for idx, i in enumerate(arr_libsvm):
#print "idx=",idx,",l=",i[0],",d=",i[1:]
if lbl_flag != i[0]:
if feature_arr and len(feature_arr) > 0:
features_list.append(np.array(feature_arr))
row_list.append(np.array(row_arr))
col_list.append(np.array(col_arr))
sample_numbers.append(cnt)
row_arr = []
col_arr = []
feature_arr = []
cnt = 0
lbl_flag += 1
for j in i[1:]:
row_arr.append(cnt)
col_arr.append(j[0] - 1)
feature_arr.append(j[1])
cnt += 1
# for last part
if len(feature_arr) > 0:
features_list.append(np.array(feature_arr))
row_list.append(np.array(row_arr))
col_list.append(np.array(col_arr))
sample_numbers.append(cnt)
#print ",features_list=",features_list
#print ",row_list=",row_list
#print ",col_list=",col_list
print "INFO: sample_numbers=", sample_numbers
col_num = len(dic_list)
print "INFO: column number: ", col_num #, ",len(max_feat_list)=",len(max_feat_list)
for i in range(0, len(features_list)):
#print "i=",i
#print "features_list=",features_list[i]
#print "row_list=",row_list[i]
#print "col_list=",col_list[i]
#print "sample_numbers=",sample_numbers[i]
sparse_mtx = csc_matrix((features_list[i], (row_list[i], col_list[i])),
shape=(sample_numbers[i], col_num))
sparse_mtx_list.append(sparse_mtx)
#print sparse_mtx_list[0]
print "INFO: sparse_mtx_list[0].shape=", sparse_mtx_list[0].shape
#print sparse_mtx_list[1]
print "INFO: sparse_mtx_list[1].shape=", sparse_mtx_list[1].shape
exclusive_feature_set_mal = []
exclusive_feature_set_clean = []
dic_feature_cnt_mal = {}
dic_feature_cnt_clean = {}
dic_score = {}
dic_cnt_mal = {}
dic_cnt_clean = {}
dic_IT_grain = {}
####################################################
####feature importance algorithms: 2 methods ####### # Only for 2 classes ???
####################################################
if len(sample_numbers) == 2:
###################################################
################## calculate probability ############
###################################################
print "INFO: =======Feature Importance(probability) ================ "
for j in range(0, col_num):
curr_col_dirty = sparse_mtx_list[0].getcol(j)
sum_col = curr_col_dirty.sum(0)
cnt_mal = sum_col.tolist()[0][0]
curr_col_clean = sparse_mtx_list[1].getcol(j)
sum_col = curr_col_clean.sum(0)
cnt_clean = sum_col.tolist()[0][0]
percnt_mal = cnt_mal / float(sample_numbers[0])
percnt_clean = cnt_clean / float(sample_numbers[1])
score_j = (percnt_mal + 1 - percnt_clean) / 2
dic_score[j + 1] = score_j
dic_cnt_clean[j + 1] = cnt_clean
dic_cnt_mal[j + 1] = cnt_mal
sorted_score = sorted(dic_score.items(),
key=operator.itemgetter(1),
reverse=True)
#print "sorted_score:", sorted_score
#print "dic_cnt_clean", dic_cnt_clean
#print "dic_cnt_mal", dic_cnt_mal
############output result########################
if os.path.exists(local_score_file):
try:
os.remove(local_score_file)
except OSError, e:
print("Error: %s - %s." % (e.local_score_file, e.strerror))
for ii in range(0, len(sorted_score)):
(feat, score) = sorted_score[ii]
#print feat, score, dic_all_columns[feat]
if dic_hash_str:
description_str = feats2strs(dic_all_columns[str(feat)],
dic_hash_str)
else:
description_str = "N/A"
print "Warning: No mapping found for feature number"
str01 = str(feat) + "\t" + str(
score) + "\t" + description_str + "\n"
with open(local_score_file, "a") as f:
f.write(str01)
########################################################
##################Information Gain (entropy)############
########################################################
print "INFO: =======Information Gain================ "
for j in range(0, col_num):
cnt_mal = dic_cnt_mal[j + 1]
cnt_clean = dic_cnt_clean[j + 1]
total_samples = sample_numbers[0] + sample_numbers[1]
p0 = float(sample_numbers[0]) / total_samples
p1 = 1 - p0
if p0 == 0 or p1 == 0:
parent_entropy = 0
else:
parent_entropy = 0 - p0 * np.log2(p0) - p1 * np.log2(p1)
if cnt_clean + cnt_mal == 0:
information_gain = 0
elif total_samples - cnt_clean - cnt_mal == 0:
information_gain = 0
else:
p0 = float(cnt_mal) / (cnt_clean + cnt_mal)
p1 = 1 - p0
if p0 == 0 or p1 == 0:
child_left_entropy = 0
else:
child_left_entropy = 0 - p0 * np.log2(p0) - p1 * np.log2(
p1)
p0 = float(sample_numbers[0] - cnt_mal) / (total_samples -
cnt_clean - cnt_mal)
p1 = 1 - p0
if p0 == 0 or p1 == 0:
child_right_entropy = 0
else:
child_right_entropy = 0 - p0 * np.log2(p0) - p1 * np.log2(
p1)
weighted_child_entropy = child_left_entropy * float(
cnt_clean +
cnt_mal) / total_samples + child_right_entropy * float(
total_samples - cnt_clean - cnt_mal) / total_samples
information_gain = parent_entropy - weighted_child_entropy
dic_IT_grain[j + 1] = information_gain
sorted_IT_gain = sorted(dic_IT_grain.items(),
key=operator.itemgetter(1),
reverse=True)
if os.path.exists(score_file_IT):
try:
os.remove(score_file_IT)
except OSError, e:
print("Error: %s - %s." % (e.score_file_IT, e.strerror))
def train(row_id_str, ds_id, hdfs_feat_dir, local_out_dir, ml_opts_jstr,
sp_master, spark_rdd_compress, spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, zipout_dir, zipcode_dir, zip_file_name,
mongo_tuples, labelnameflag, fromweb, src_filename, jobname,
model_data_folder):
# create zip files for Spark workers ================= ================
zip_file_path = ml_build_zip_file(zipout_dir,
zipcode_dir,
zip_file_name,
prefix='zip_feature_util')
print "INFO: zip_file_path=", zip_file_path
#data_folder = hdfs_feat_dir + "/"
#local_out_dir = local_out_dir + "/"
if not os.path.exists(local_out_dir):
os.makedirs(local_out_dir)
# ML model filename ====
model_fname = os.path.join(model_data_folder, row_id_str + '.pkl')
print "INFO: model_data_folder=", model_data_folder
# create out folders and clean up old model files ====
ml_util.ml_prepare_output_dirs(row_id_str, local_out_dir,
model_data_folder, model_fname)
# init Spark context ====
sc = ml_util.ml_get_spark_context(sp_master, spark_rdd_compress,
spark_driver_maxResultSize,
sp_exe_memory, sp_core_max, jobname,
[zip_file_path])
# start here =================================================================== ===============
t0 = time()
### load libsvm file: may or may not be PCA-ed ###
libsvm_data_file = os.path.join(hdfs_feat_dir, src_filename)
print "INFO: libsvm_data_file=", libsvm_data_file
# feature count is a variable if PCA
feature_count = 0
# samples_rdd may be from PCAed data
# load sample RDD from text file
# format (LabeledPoint,hash) from str2LabeledPoint_hash()
samples_rdd, feature_count = zip_feature_util.get_sample_rdd(
sc, libsvm_data_file, feature_count, '')
# collect all data to local for processing ===============
all_data = samples_rdd.collect()
total_sample_count = len(all_data)
# 2-D array, may be PCAed
features_list = [x.features.toArray() for x, _ in all_data]
# label array
labels_list_all = [x.label for x, _ in all_data]
# hash array
hash_list_all = [x for _, x in all_data]
# convert to np array
features_array_reduced = np.array(features_list)
hash_list_all = np.array(hash_list_all)
labels_list_all = np.array(labels_list_all)
true_label_array = np.array(labels_list_all, dtype=np.int8)
print "INFO: total_sample_count=", total_sample_count
print "INFO: features_array_reduced.shape=", features_array_reduced.shape
print "INFO: labels_list_all.shape=", labels_list_all.shape
print "INFO: true_label_array.shape=", true_label_array.shape
t1 = time()
print 'INFO: data generating time: %f' % (t1 - t0)
###############################################
########## build learning model ###############
###############################################
### parse parameters and generate the model ###
(model, alg, n_clusters) = parse_para_and_get_model(ml_opts_jstr)
if model is None:
return
labels_kmeans = None
#### fit the model to training dataset ####
try:
model.fit(features_array_reduced)
labels_kmeans = model.labels_ #'numpy.ndarray'
except:
print "ERROR: Error in model.fit(): ", "model=", model, ", sys.exc_info:", sys.exc_info(
)[0]
return
#### save clf for future use ####
#joblib.dump(model, model_data_folder + row_id_str+'.pkl')
joblib.dump(model, model_fname)
#print "**model:intercept***"
#print clf.intercept_
print "INFO: model type=", type(model), " model=", model
###################################################
### generate label names (family names) ###########
### connect to database to get the column list which contains all column number of the corresponding feature####
###################################################
if labelnameflag == 1:
key = "dic_name_label"
jstr_filter = '{"rid":' + row_id_str + ',"key":"' + key + '"}'
jstr_proj = '{"value":1}'
# get parent dataset's data
if ds_id != row_id_str:
jstr_filter = '{"rid":' + ds_id + ',"key":"' + key + '"}'
doc = query_mongo.find_one_t(mongo_tuples, jstr_filter, jstr_proj)
dic_list = doc['value']
label_dic = {}
for i in range(0, len(dic_list)):
for key in dic_list[i]:
label_dic[dic_list[i][key]] = key.encode('UTF8')
print "INFO: label_dic:", label_dic
else:
label_dic = {}
label_set = set(labels_list_all)
for label_value in label_set:
label_dic[int(label_value)] = str(int(label_value))
print "INFO: generated label_dic:", label_dic
labels_list = []
for key in sorted(label_dic):
labels_list.append(label_dic[key])
print "INFO: labels_list=", labels_list
#Adjusted Mutual Information between two clusterings
amis = adjusted_mutual_info_score(labels_list_all, labels_kmeans)
print "INFO: Adjusted_mutual_info_score=", amis
#Similarity measure between two clusterings
ars = adjusted_rand_score(labels_list_all, labels_kmeans)
print "INFO: Adjusted_rand_score=", ars
###################################################
#######plot histogram ####
###################################################
plot_col_num = int(math.ceil(math.sqrt(n_clusters)))
figsize = (4 * plot_col_num,
3 * int(math.ceil(n_clusters * 1.0 / plot_col_num)))
print "INFO: labels_list_all.shape=", labels_list_all.shape, "labels_kmeans.shape=", labels_kmeans.shape
print "INFO: labels_list_all t=", type(
labels_list_all), "labels_kmeans t=", type(labels_kmeans)
print "INFO: n_clusters=", n_clusters, ",label_dic=", label_dic
print "INFO: plot_col_num=", plot_col_num, ",figsize=", figsize, ",local_out_dir=", local_out_dir
# kmeans histogram
_, p_true = ml_plot_kmeans_histogram_subfigures(labels_list_all,
labels_kmeans,
n_clusters,
names=label_dic,
plot_col_num=plot_col_num,
figsize=figsize,
folder=local_out_dir,
rid=row_id_str)
# normalized kmeans histogram
_, p_true_norm = ml_plot_kmeans_histogram_subfigures(
labels_list_all,
labels_kmeans,
n_clusters,
names=label_dic,
plot_col_num=plot_col_num,
figsize=figsize,
normalize=True,
folder=local_out_dir,
rid=row_id_str)
####plot "reverse" histogram with labels ####
#num_bars = len(np.unique(labels_list_all))
num_bars = max(labels_list_all) + 1
figsize = (4 * plot_col_num,
3 * int(math.ceil(num_bars * 1.0 / plot_col_num)))
_, p_cluster = ml_plot_kmeans_histogram_subfigures(
labels_kmeans,
labels_list_all,
num_bars,
names=label_dic,
plot_col_num=plot_col_num,
figsize=figsize,
reverse=True,
folder=local_out_dir,
rid=row_id_str)
#### plot dot figures ####
#mtx_label = model.labels_
mtx_center = model.cluster_centers_
# dot plot for Kmeans ===========
filename = os.path.join(local_out_dir, row_id_str + '_cluster.png')
filename_3d = os.path.join(local_out_dir, row_id_str + '_cluster_3d.json')
ml_plot_kmeans_dot_graph_save_file(features_array_reduced,
labels_kmeans,
mtx_center,
n_clusters,
figsize=(10, 7),
filename=filename,
title='KMeans',
filename_3d=filename_3d)
#print "features_array_reduced s=",features_array_reduced.shape
# dot plot for True Labels ===========
filename = os.path.join(local_out_dir, row_id_str + '_cluster_tl.png')
filename_3d = os.path.join(local_out_dir,
row_id_str + '_cluster_3d_tl.json')
ml_plot_kmeans_dot_graph_save_file(features_array_reduced,
true_label_array,
mtx_center,
n_clusters,
figsize=(10, 7),
filename=filename,
title='True Labels',
filename_3d=filename_3d)
dataset_info = {
"training_fraction": 1,
"class_count": n_clusters,
"dataset_count": total_sample_count
}
# only update db for web request ===========
if fromweb == "1":
#print "database update"
str_sql="UPDATE atdml_document set accuracy = '" \
+"', status = 'learned', processed_date ='"+str(datetime.datetime.now()) \
+"', total_feature_numb='"+str(feature_count) \
+"', perf_measures='{}" \
+"', dataset_info='"+json.dumps(dataset_info) \
+"' where id="+row_id_str
ret = exec_sqlite.exec_sql(str_sql)
print "INFO: Data update done! ret=", str(ret)
else:
print "INFO: accuracy = '" + str(accuracy * 100) + "%"
t1 = time()
print 'INFO: running time: %f' % (t1 - t0)
#print 'Finished!'
return 0
