def get_types_and_membership(predictionrun_id=None,
top_k_candidates=5,
model_dir=None):
if model_dir is None:
print 'get_types_and_membership> model_dir should not be None'
return []
if predictionrun_id is None:
print 'get_types_and_membership> predictionrun_id should not be None'
return []
predictionrun = PredictionRun.objects.filter(id=predictionrun_id)
if len(predictionrun) != 1:
print 'get_types_and_membership> predictionrun_id is not longer exists'
return []
predictionrun = predictionrun[0]
model, types = learning.load_model(model_dir)
types = np.array(types)
list_of_mem_with_types = []
print 'mem with types'
for m in Membership.objects.filter(prediction_run=predictionrun):
mem_with_types = {}
mems = m.get_values_as_numpy()
mems_idxs = mems.argsort(
)[::
-1][:
top_k_candidates] # idxs sorted from largest (value not largest index) to smallest
mems = mems[mems_idxs]
mems *= 100
mem_with_types["typesscores"] = zip(mems.tolist(),
types[mems_idxs].tolist())
mem_with_types["column_no"] = m.column_no
mem_with_types["file_name"] = m.file_name
list_of_mem_with_types.append(mem_with_types)
return list_of_mem_with_types
def predict_files(predictionrun_id=None,
model_dir=None,
files=[],
original_uploaded_filenames=[],
has_header=False):
"""
:param predictionrun_id:
:param model_dir: the dir of the FCM model csv file abs dir
:param files: list of files to be predicted
:return:
"""
if predictionrun_id is None:
print "predict_files> predictionrun_id should not be None"
return
if model_dir is None:
print "predict_files> model_dir should not be None"
return
if len(files) != len(original_uploaded_filenames):
print "predict_files> number of files (%d) does not equal original_uploaded_filenames (%d)" % \
(len(files), len(original_uploaded_filenames))
return
print "original uploaded files:"
print original_uploaded_filenames
update_func = partial(update_predictionrun_progress_for_partial,
predictionrun_id)
update_predictionrun_state(predictionrun_id=predictionrun_id,
new_progress=0,
new_state=PredictionRun.RUNNING)
model, types = learning.load_model(model_dir)
num_of_files = len(files)
for idx, fname in enumerate(files):
update_predictionrun_state(predictionrun_id=predictionrun_id,
new_notes='predicting columns in file: ' +
fname.split('/')[-1].strip()[:-4])
data, meta_data = data_extraction.data_and_meta_from_a_mixed_file(
file_name=fname,
has_header=has_header,
original_file_name=original_uploaded_filenames[idx])
print "predict_files> extracted data shape is %s " % str(data.shape)
u = learning.predict(model=model,
data=data,
meta_data=meta_data,
update_func=update_func)
predictionrun = PredictionRun.objects.filter(id=predictionrun_id)
if len(predictionrun) == 1:
predictionrun = predictionrun[0]
file_column_list = [{
"file_name": fc["type"].split(' , ')[0],
"column_no": fc["type"].split(' , ')[1]
} for fc in meta_data]
predictionrun.add_memberships(u, file_column_list)
else:
update_predictionrun_state(
predictionrun_id=predictionrun_id,
new_notes="predictionrun_id is not longer exists",
new_state=PredictionRun.STOPPED)
return
predictionrun = PredictionRun.objects.filter(id=predictionrun_id)
if len(predictionrun) == 1:
predictionrun = predictionrun[0]
predictionrun.set_types(types)
print "setting types"
print types
else:
update_predictionrun_state(
predictionrun_id=predictionrun_id,
new_notes="predictionrun_id is not longer exists",
new_state=PredictionRun.STOPPED)
return
update_predictionrun_state(predictionrun_id=predictionrun_id,
new_progress=100,
new_state=PredictionRun.COMPLETE,
new_notes='')
parser.add_argument("checkpoint_path", type=str, help="path to the checkpoint")
parser.add_argument("--mapping_path",
type=str,
default="",
help="path to the checkpoint (Default: No_Mapping)")
parser.add_argument("--top_k",
type=int,
default="3",
help="path to the checkpoint (Default: 3)")
parser.add_argument('--gpu',
action='store_true',
help="Use GPU for training (Default: False)")
args = parser.parse_args()
#load name mapping from json file
cat_to_name = utility.load_mapping(
args.mapping_path) if args.mapping_path != "" else {}
#load model from checkpoint
model = learning.load_model(args.checkpoint_path)
#predict classes and propabilities
ps, preds = learning.predict(args.img_path, model, args.top_k, args.gpu)
#map classes to names
preds = [cat_to_name[pred] for pred in preds] if cat_to_name else preds
#print results
for name, prob in zip(preds, ps[-1, :]):
print("Class: {}, Propability: {}%".format(name, prob * 100))