def _update_dict_for_user(user, dict_for_user):
"""
Write the dict into user's specific file
:param user:
:type user:
:return:
:rtype:
"""
if FILE_MODE:
dict_to_dump = {user: dict_for_user}
filename = _get_file_name(user)
with open(filename, 'wb') as outp:
json.dump(Serializable.dumps(dict_to_dump), outp)
else:
filename = os.path.join(os.getcwd(), 'results', WIKINAME,
'user_graph.json')
with open(filename, 'rb') as inp:
user_dict = Serializable.loads(json.load(inp))
user_dict[user] = dict_for_user
with open(filename, 'wb') as outp:
json.dump(Serializable.dumps(user_dict), outp)
def dumps(self):
d = dict(
in_features=self.in_features,
out_features=self.out_features,
weight=self.weight.data.cpu().numpy(),
bias=None if self.bias is None else self.bias.data.cpu().numpy(),
)
return Serializable.dumps(d)
def dumps(self):
"""Writes itself to a string."""
# Creates a dictionary
d = dict(
in_features=self.in_features,
out_features=self.out_features,
min_input=self.w.lower_bound,
max_input=self.w.upper_bound,
min_slope=self.u.lower_bound,
max_slope=self.u.upper_bound,
modinf=self.modinf,
regular_deriv=self.regular_deriv,
andor=self.andor,
andor01=self.andor01.cpu().numpy(),
u=self.u.data.cpu().numpy(),
w=self.w.data.cpu().numpy(),
)
return Serializable.dumps(d)
def _create_label_file(labels, name_suffix):
filename = os.path.join(os.getcwd(), 'results', WIKINAME,
'labels_%s.json' % (name_suffix))
with open(filename, 'wb') as outp:
outp.write(Serializable.dumps(labels))
pass
for label in all_labels:
label = str(label)
# total_prec_list[label].append(precision_dict[label])
# total_recall_list[label].append(recall_dict[label])
# total_avg_recall_list.append(np.mean(recall_list))
# total_f1_list[label].append(_f1(precision_dict[label], recall_dict[label]))
for keyname in ['prec', 'rec', 'f1']:
if not results.has_key(keyname):
results[keyname] = {}
if not results[keyname].has_key(label):
results[keyname][label] = []
if not results.has_key('avg_rec'):
results['avg_rec'] = []
results['prec'][label].append(precision_dict[int(label)])
results['rec'][label].append(recall_dict[int(label)])
results['f1'][label].append(
_f1(precision_dict[int(label)], recall_dict[int(label)]))
results['avg_rec'].append(np.mean(recall_list))
with open(results_file, 'wb') as outp:
outp.write(Serializable.dumps(results))
# results = dict(prec = total_prec_list, rec = total_recall_list, f1=total_f1_list, avg_rec=total_avg_recall_list)
#
# with open('results_depth_%d.json'%(DEPTH), 'wb') as outp:
# outp.write(Serializable.dumps(results))
# print "Depth was: %d"%(DEPTH)
def dumps(self):
d = dict(args=self.args.__dict__,
layers=[l.dumps() for l in self.layers])
return Serializable.dumps(d)
def train_nn_using_k_lstm_bit(train_dict,
k=None,
N=1000,
quality=True,
fix_bit_val=None,
store=False,
picklefile=os.path.join(os.getcwd(), 'results',
'temp_model.pkl'),
weighted_learning=False,
balanced=True):
"""
Train the LSTM and NNet combination using training dict.
:param train_dict: dict containing entries of revisions per user
:param k: Number of bits to be used from LSTM
:param N: Number of iterations for network to train. Default is 1000
:param quality: Boolean to control if working on quality, otherwise existence
:param fix_bit_val: Fixed value of bit to be used if only that bit should be passed to NN
:param store: Boolean to decide whether to store result in pickle
:param picklefile: Pickle filename
:param weighted_learning: Boolean to control whether learning is weighted or not
:param balanced: Boolean to control whether results should be balanced before use or not
:rtype tuple
:return: Returns a tuple consisting of lstm and neural net (lstm, nnet)
"""
train_items = train_dict.items()
if balanced:
train_items = _rebalance_data(train_items)
# Send for training using k as no. of bits to use
print "\n==Starting training== (Using %r iterations) and k=%r" % (N, k)
print "Statuses-- Weighted: %r, Balanced %r" % (weighted_learning,
balanced)
t_start = time.clock()
(lstm_out, nn_out), errors = _train_nn_with_k_lstm_bits(
train_items,
k=k,
N=N,
fix_bit_val=fix_bit_val,
weighted_learning=weighted_learning,
quality=quality)
print "Training completed in %r seconds" % (time.clock() - t_start)
# Store the trained model into a pickle if store is True
if store:
file_basic_name = 'trained_lstm_%r_nn_%r_%r' % (
k, N, "weighted" if weighted_learning else "unweighted")
serialize_file_lstm = os.path.join(os.getcwd(), 'results',
file_basic_name + 'lstm.json')
serialize_file_nn = os.path.join(os.getcwd(), 'results',
file_basic_name + 'nn.json')
from json_plus import Serializable
ser_result_lstm = Serializable.dumps(lstm_out)
ser_result_nn = Serializable.dumps(nn_out)
with open(serialize_file_lstm, 'wb') as output:
json.dump(ser_result_lstm, output)
with open(serialize_file_nn, 'wb') as output:
json.dump(ser_result_nn, output)
# Store the (lstm, nnet) type result into a pickle
with open(picklefile, 'wb') as output:
pickle.dump((lstm_out, nn_out), output, pickle.HIGHEST_PROTOCOL)
return (lstm_out, nn_out)
def write_model(m, out_fn):
with open(out_fn, 'w') as f:
d = dict(encoding=0, model=m)
f.write(Serializable.dumps(d))
# with open(user_contrib_file, 'rb+') as inp:
# user_contribs = json.load(inp)
# else:
# user_contribs = {}
# get_files(base_dir, user_graph, user_contribs)
# get_split_files(base_dir)
if FILE_MODE:
get_files(base_dir)
else:
user_graph = get_user_dict(base_dir)
filename = os.path.join(os.getcwd(), 'results', WIKINAME,
'user_graph.json')
with open(filename, 'wb') as outp:
outp.write(Serializable.dumps(user_graph))
# with open(user_graph_file, 'wb+') as output:
# json.dump(user_graph, output)
#
# with open(user_contrib_file, 'wb+') as output:
# json.dump(user_contribs, output)
#
# # pprint(user_graph)
# print "Users in graph", len(user_graph.keys())
print "None counter", NONECTR
# for i in random.sample(user_graph.keys(),10):
# pprint(user_graph[i])
# print "--------"