def collect_reviews(gc, all_ids):
'''
all_ids : list of user_ids for which reviews are needed.
'''
global book_data, author_data
for user_id in all_ids:
friend = GoodreadsUserExtended.fromBaseUser(gc.user(user_id))
print_log("{} ".format((get_name(friend.name), friend.gid)))
user_data = {}
user_data[friend.gid] = {'name':friend.name, 'books':{}}
user_reviews = friend.reviews(page='all')
for i, review in enumerate(user_reviews):
try:
book = GoodreadsBook(review.book, gc)
if isinstance(book.gid, OrderedDict):
book.gid = dict(book.gid)['#text']
if not book_data.has_key(book.gid):
book_data[book.gid] = extract_book_info(book)
user_data[friend.gid]['books'][book.gid] = review.rating
for author in book.authors:
if not author_data.has_key(author.gid):
author_data[author.gid] = extract_author_info(author)
except:
print_log("error : review for friend_id={} review_dict {}".format(friend.gid,
review._review_dict))
friend_name = '_'.join(get_name(friend.name).split()).lower()
fname = DATA_DIR + 'user_{}_{}.json'.format(friend_name, friend.gid)
save_dictionary(user_data, fname)
save_dictionary(book_data, DATA_DIR + 'book_data.json')
save_dictionary(author_data, DATA_DIR + 'author_data.json')
def load_dictionary(fpath):
try:
with open(fpath) as f:
data = json.load(f)
return data
except:
print_log('error : could not load file at {}'.format(fpath))
def do_inference(onnx_model):
import time
num_tests = 100
num_images = 100
total_tests = 0
total_elapsed = 0
#import ipdb as pdb; pdb.set_trace()
sess = rt.InferenceSession(onnx_model)
input_name = sess.get_inputs()[0].name
pass_cnt = 0
utility.print_banner("Starting Inference Engine")
for test_id in range(0, num_tests):
images, labels = load_images(num_images)
inf_start = time.time()
pred_onx = sess.run(None, {input_name: images.astype(np.float32)})[0]
inf_end = time.time()
elapsed_time = inf_end - inf_start
total_elapsed += elapsed_time
utility.print_log("batch {} took {:2.4f}ms to complete".format(
test_id, elapsed_time * 1000))
for i in range(0, num_images):
pred = pred_onx[i].argmax()
res = "FAIL"
if labels[i] == pred:
res = "PASS"
pass_cnt += 1
total_tests += 1
utility.print_log("actual={} pred={} res={}".format(
labels[i], pred, res),
verbosity="VERB_HIGH")
avg_inf = 1000.0 * float(total_elapsed) / total_tests
utility.print_banner(
"Accuracy = {}% out of {} tests, avg inference={:2.4}ms per image".
format(100 * pass_cnt / (float(num_images) * num_tests),
float(num_images) * num_tests, avg_inf))
def save_output(output_vals, output_filename):
file_object = open(output_filename, 'w')
for i in range(len(output_vals)):
temp = ""
for j in range(output_vals[i].shape[0]):
temp += str(int(output_vals[i][j]))
file_object.write(temp + "\n")
file_object.close()
util.print_log("saving outputs to {0}".format(output_filename))
def convertBin(value):
if (value == 1):
return "01"
elif (value == -1):
return "11"
elif (value == 0):
return "00"
else:
return "output_size"
util.print_log("only ternary weights of -1, 0 and 1 is supported",
id_str="ERROR")
sys.exit()
def run_model(windows, W1, W2, W3, b1, b2, b3, verbosity="VERB_HIGH"):
AC1 = hw.AccelCore(l1_input, l2_input)
AC2 = hw.AccelCore(l2_input, l3_input)
AC3 = hw.AccelCore(l3_input, output_size)
out = []
for i in range(windows.shape[0]):
temp_out = AC1.forward(W1, windows[i].flatten(), b1)
temp_out = AC2.forward(W2, temp_out, b2)
out.append(AC3.forward(W3, temp_out, b3))
if ((i % 100) == 0):
util.print_log("Iteration {0}".format(i), verbosity=verbosity)
return out
def load_book_data_as_input(folder_path):
book_data = load_dictionary(glob.glob(folder_path + 'book_data.json')[0])
dataset= []
ids = []
for key, value in book_data.items():
descr = value['description']
if descr:
descr = clean_html(descr.lower())
else:
descr = get_name(value['title'].lower())
dataset.append(descr)
ids.append(key)
print_log("dataset size : {}".format(len(dataset)))
return ids, dataset
def book_id2attr_map(folder_path, attributes=['title']):
book_data = load_dictionary(glob.glob(folder_path + 'book_data.json')[0])
print_log('total books loaded = {}'.format(len(book_data)))
d_list = [dict() for i in range(len(attributes))]
for key, value in book_data.items():
for idx, d in enumerate(d_list):
if attributes[idx]=='description':
descr = value[attributes[idx]]
if descr:
descr = clean_html(descr.lower())
d[key] = descr
else:
d[key] = value[attributes[idx]]
return d_list
def load_user_data(folder_path, book_id2attr = None):
'''
Returns:
dataset = list of docs. Each doc is a string.
vocabulary = set of unique words
'''
all_user_files = glob.glob(folder_path +'user_*.json')
dataset = []
for fpath in all_user_files:
d = load_dictionary(fpath)
books = d.values()[0]['books'].keys()
doc = create_user_doc(books, book_id2attr)
dataset.append(doc)
print_log("dataset size : {}".format(len(dataset)))
return dataset
def optimize_model(expr_name, onnx_file, model_xml, weight_bin):
run_opt = False
if (model_xml == None):
util.print_log("Could not find xml model", id_str="warning")
run_opt = True
if (weight_bin == None):
util.print_log("Could not find binary weights", id_str="warning")
run_opt = True
if run_opt:
util.print_banner(
"Running OpenVino optimizer on {0}".format(onnx_file),
color='green',
verbosity="VERB_LOW")
cmd = "python {0}/deployment_tools/model_optimizer/mo.py --input_model={1} --model_name {2}".format(
openvino_inst_path, onnx_file, expr_name)
util.run_command(cmd, verbosity="VERB_LOW")
model_xml, weight_bin = expr_name + ".xml", expr_name + ".bin"
# load model
# import ipdb as pdb; pdb.set_trace()
return model_xml, weight_bin
def optimize_model(expr_name, onnx_file, model_xml, weight_bin, opt_ops=""):
# import ipdb as pdb; pdb.set_trace()
run_opt = False
options = ""
if (model_xml == None):
util.print_log("No xml model was provided", id_str="warning")
run_opt = True
if (weight_bin == None):
util.print_log("No binary weight file was provided", id_str="warning")
run_opt = True
if run_opt:
util.print_banner("Running OpenVino optimizer on {0}".format(onnx_file), color='green', verbosity="VERB_LOW")
options += "--input_model={0} ".format(onnx_file)
options += "--model_name {0} ".format(expr_name)
options += (" --"+opt_ops[0]) if len(opt_ops)==1 else "".join(" --"+e for e in opt_ops)
cmd = "python {0}/deployment_tools/model_optimizer/mo_onnx.py {1}".format(openvino_inst_path, options)
util.run_command(cmd, verbosity="VERB_LOW")
model_xml, weight_bin = expr_name+".xml", expr_name+".bin"
# load model
# import ipdb as pdb; pdb.set_trace()
return model_xml, weight_bin
def reviews(self, page=1):
"""
Get all books and reviews on user's shelves
page = 'all' if all reviews are requested.
Note:
reviews() function of Goodreads.user class is a bit buggy.
If there is only one review to return, it is not handled
properly there.
"""
all_reviews = []
start_page_index = page
if page == 'all':
start_page_index = 1
while True:
try:
resp = self._client.session.get("/review/list.xml", {
'v': 2,
'id': self.gid,
'page': start_page_index
})
if isinstance(resp['reviews']['review'], list):
all_reviews += [
review.GoodreadsReview(r)
for r in resp['reviews']['review']
]
else:
all_reviews += [
review.GoodreadsReview(resp['reviews']['review'])
]
except:
break
start_page_index += 1
if page != 'all':
break
print_log('total reviews collected from id={}, name={} :{}'.format(
self.gid, get_name(self.name), len(all_reviews)))
if len(all_reviews) == 0:
print_log("Returning 0 reviews: {}".format(all_reviews))
return all_reviews
def get_rtl_files(f_file):
sv_rtl = ""
vhdl_rtl = ""
v_rtl = ""
with open(f_file, 'r') as f:
rtls = f.readlines()
for rtl in rtls:
rtl = rtl.replace("\n", "")
if rtl != "":
if rtl.lower().endswith(".vhdl") or rtl.lower().endswith(
".vhd"):
vhdl_rtl += "{0} ".format(rtl)
elif rtl.lower().endswith(".sv") or rtl.lower().endswith(
".svh"):
sv_rtl += "{0} ".format(rtl)
elif rtl.lower().endswith(".v") or rtl.lower().endswith(".vh"):
v_rtl += "{0} ".format(rtl)
else:
util.print_log("unsupported file format: {0}".format(rtl),
"ERROR",
verbosity="VERB_LOW")
sys.exit()
# import ipdb as pdb; pdb.set_trace()
return sv_rtl, v_rtl, vhdl_rtl
w_cnt += 1
return window
if __name__ == '__main__':
args = parse_args()
verbosity = args['verbosity']
verbosity = verbosity if verbosity != None else "VERB_HIGH"
input_img_width = int(args['input_img_width'])
input_img_height = int(args['input_img_height'])
kernel_size = int(args['kernel_size'])
input_img = open(args['input_img'], "r").read()
output_window = open(args['output_window'], "r").read()
util.print_banner("Testing Window Slider Output ", verbosity='VERB_LOW')
util.print_log("input image: {0} ".format(args['input_img']))
util.print_log("simulation output: {0} ".format(args['output_window']))
input_img = input_img.replace("\n", ",").split(",")
input_img = input_img[:-1]
input_img = [int(x) for x in input_img]
input_img = np.asarray(input_img)
input_img = input_img.reshape((input_img_height, input_img_width))
sw_output = gen_window(image=input_img,
w_height=kernel_size,
w_width=kernel_size,
stride=1)
output_window = output_window.replace("1'b", "").replace("'{", "").replace(
"}", "").split("\n")
total_num_windows = ((input_img_height - kernel_size) + 1) * (
(input_img_width - kernel_size) + 1)
gui = args['gui']
svseed = args['svseed']
coverage = args['coverage']
debug = args['debug']
waveform = args['waveform']
clean = args['clean']
silence = args['silence']
verbosity = args['verbosity']
compileonly = args['compileonly']
elabonly = args['elabonly']
if verbosity is None:
verbosity = 'VERB_LOW'
if util.get_platform(verbosity=verbosity) != "linux":
util.print_log("This script works only on a Linux platform",
"ERROR",
verbosity="VERB_LOW")
sys.exit()
if clean:
util.print_banner("Cleaning project", verbosity=verbosity)
util.clean_proj(files_to_clean)
if not os.path.exists(result_dir):
util.print_log("Creating a result directory in {0}".format(result_dir),
"INFO",
verbosity="VERB_LOW")
os.makedirs(result_dir)
if simulator == None:
util.print_log("You need to provide Simulator name",
"ERROR",
verbosity="VERB_LOW")
sys.exit()
def get_vocabulary(dataset):
vocabulary = list(set(" ".join(dataset).split()))
print_log("vocabulary size : {}".format(len(vocabulary)))
return vocabulary
def load_params(img_npy, w1_npy, w2_npy, w3_npy, b1_npy, b2_npy, b3_npy,
w_width, w_height, stride):
# import ipdb as pdb; pdb.set_trace()
util.print_log("loading {0}...".format(img_npy))
image = np.load(img_npy)
windows = (gen_window(image, w_width, w_height, stride))
util.print_log("loading {0}...".format(w1_npy))
w1 = np.load(w1_npy)
util.print_log("loading {0}...".format(w2_npy))
w2 = np.load(w2_npy)
util.print_log("loading {0}...".format(w3_npy))
w3 = np.load(w3_npy)
util.print_log("loading {0}...".format(b1_npy))
b1 = np.load(b1_npy)
util.print_log("loading {0}...".format(b2_npy))
b2 = np.load(b2_npy)
util.print_log("loading {0}...".format(b3_npy))
b3 = np.load(b3_npy)
return windows, w1, w2, w3, b1, b2, b3
#=======================================================================
# Main
#=======================================================================
if __name__ == '__main__':
cmd_to_run = ""
__args__ = parse_args()
project_name = __args__['project_name']
path_for_proj = __args__['path_proj']
path_fpga = __args__['path_fpga']
verbosity = __args__['vebosity']
util.print_banner("Creating {0} Project".format(project_name),
verbosity="VERB_LOW")
# import ipdb as pdb; pdb.set_trace()
if path_for_proj == None:
default_path_for_proj = os.getcwd() + "/"
util.print_log("Using current path for creating project: {0}".format(
default_path_for_proj))
path_for_proj = default_path_for_proj
if path_fpga == None:
util.print_log("Using default path to fpga directory {0}".format(
default_path_to_fpga_dir))
path_fpga = default_path_to_fpga_dir
# Check if project has already been created
if os.path.isdir("{0}{1}".format(path_for_proj, project_name)):
util.print_log("Project path {0}{1} already exist!".format(
path_for_proj, project_name),
id_str="ERROR")
sys.exit()
if os.getcwd() not in path_for_proj:
proj_dir = os.getcwd() + "/" + path_for_proj + project_name
else:
proj_dir = path_for_proj + project_name
for author in book.authors:
if not author_data.has_key(author.gid):
author_data[author.gid] = extract_author_info(author)
except:
print_log("error : review for friend_id={} review_dict {}".format(friend.gid,
review._review_dict))
friend_name = '_'.join(get_name(friend.name).split()).lower()
fname = DATA_DIR + 'user_{}_{}.json'.format(friend_name, friend.gid)
save_dictionary(user_data, fname)
save_dictionary(book_data, DATA_DIR + 'book_data.json')
save_dictionary(author_data, DATA_DIR + 'author_data.json')
if __name__ == "__main__":
logging.basicConfig(filename='result.log', level=logging.INFO)
gc = client.GoodreadsClient(APP_KEY, APP_SECRET)
#print_log(gc.session.access_token)
#print_log(gc.session.access_token_secret)
gc.authenticate(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
user = gc.auth_user()
print_log("Hello I am {} with id = {}".format(user.name, user.gid))
user = GoodreadsUserExtended.fromBaseUser(user)
friend_list = user.friends(all=True)
all_ids = [u.gid for u in friend_list+[user]]
collect_reviews(gc, all_ids)
print_log('total books = {}'.format(len(book_data)))
print_log('total authors = {}'.format(len(author_data)))
print_log('total users = {}'.format(len(all_ids)))
exit()
def get_reviews(gc, user_id='29400625'):
user = GoodreadsUserExtended.fromBaseUser(gc.user(user_id))
reviews = user.reviews()
for review in reviews:
print_log(easy_json(review.book))
return
def save_dictionary(data, fpath):
with open(fpath, 'w') as f:
json.dump(data, f)
print_log("Saved as {}".format(fpath))
top_level = args['top_level']
files = args['files']
gui = args['gui']
svseed = args['svseed']
coverage = args['coverage']
debug = args['debug']
waveform = args['waveform']
clean = args['clean']
# import ipdb as pdb; pdb.set_trace()
silence = args['silence']
verbosity = args['verbosity']
if verbosity is None:
verbosity = 'VERB_LOW'
if util.get_platform(verbosity=verbosity) != "linux":
util.print_log("This script works only on a Linux platform",
"ERROR",
verbosity="VERB_LOW")
sys.exit()
if clean:
util.print_banner("Cleaning project", verbosity=verbosity)
util.clean_proj(files_to_clean)
if not os.path.exists(result_dir):
util.print_log("Creating a result directory in {0}".format(result_dir),
"INFO",
verbosity="VERB_LOW")
os.makedirs(result_dir)
if simulator == None:
util.print_log("You need to provide Simulator name",
"ERROR",
verbosity="VERB_LOW")
sys.exit()
get_name(d.values()[0]['name']),\
d.values()[0]['books'].keys()
#TODO: fetch indices of books user has read.
if __name__ == "__main__":
logging.basicConfig(filename='result.log', filemode='w',level=logging.INFO)
n_topics = 10
n_top_words = 10
n_features = 1000
min_df = 2
max_df = 0.2
max_iter = 10
print_log('n_topics = {}, n_top_words = {}'.format(n_topics, n_top_words))
print_log('min_df = {}, max_df = {}'.format(min_df, max_df))
book_id2title, book_id2description = book_id2attr_map(DATA_DIR, attributes=['title','description'])
# dataset = load_user_data(DATA_DIR, book_id2description)
book_ids, dataset = load_book_data_as_input(DATA_DIR)
vocabulary = get_vocabulary(dataset)
# print_log(book_id2description['1362'])
print("Extracting tf features for LDA...")
tf_vectorizer = CountVectorizer(max_df=max_df, min_df=min_df,
max_features=n_features,
#vocabulary=vocabulary,
stop_words='english')
t0 = time()
tf = tf_vectorizer.fit_transform(dataset)
def print_top_words(model, feature_names, n_top_words, book_data = None):
for topic_idx, topic in enumerate(model.components_):
print_log("Topic #%d:" % topic_idx)
print_log(", ".join([get_name(feature_names[i])
for i in topic.argsort()[:-n_top_words - 1:-1]]))
util.print_log("saving outputs to {0}".format(output_filename))
if __name__ == '__main__':
args = parse_args()
output_filename = args['output_filename']
operation_type = args['operation_type']
verbosity = args['verbosity']
if output_filename == None:
output_filename = "output.txt"
if operation_type == None:
operation_type = "re_run"
if verbosity == None:
verbosity = "VERB_HIGH"
util.print_banner("Running HW simulator", verbosity="VERB_LOW")
if operation_type.lower() == "re_run":
util.print_log("re-running the model...", verbosity="VERB_LOW")
windows, W1, W2, W3, b1, b2, b3 = hw.load_params(
"image.npy", "w1.npy", "w2.npy", "w3.npy", "b1.npy", "b2.npy",
"b3.npy", w_width, w_height, stride)
elif operation_type.lower() == "normal":
util.print_log("running in normal model...", verbosity="VERB_LOW")
windows, W1, W2, W3, b1, b2, b3 = hw.gen_random_params(
input_file_name, image_h, image_w, w_width, w_height, stride,
l1_input, l2_input, l3_input, output_size)
hw.save_model(W1, W2, W3)
hw.gen_sv_weight_file("weight.svh", W1, W2, W3, b1, b2, b3)
out = run_model(windows, W1, W2, W3, b1, b2, b3, verbosity)
save_output(out, output_filename)
