def collect_reviews(gc, all_ids): ''' all_ids : list of user_ids for which reviews are needed. ''' global book_data, author_data for user_id in all_ids: friend = GoodreadsUserExtended.fromBaseUser(gc.user(user_id)) print_log("{} ".format((get_name(friend.name), friend.gid))) user_data = {} user_data[friend.gid] = {'name':friend.name, 'books':{}} user_reviews = friend.reviews(page='all') for i, review in enumerate(user_reviews): try: book = GoodreadsBook(review.book, gc) if isinstance(book.gid, OrderedDict): book.gid = dict(book.gid)['#text'] if not book_data.has_key(book.gid): book_data[book.gid] = extract_book_info(book) user_data[friend.gid]['books'][book.gid] = review.rating for author in book.authors: if not author_data.has_key(author.gid): author_data[author.gid] = extract_author_info(author) except: print_log("error : review for friend_id={} review_dict {}".format(friend.gid, review._review_dict)) friend_name = '_'.join(get_name(friend.name).split()).lower() fname = DATA_DIR + 'user_{}_{}.json'.format(friend_name, friend.gid) save_dictionary(user_data, fname) save_dictionary(book_data, DATA_DIR + 'book_data.json') save_dictionary(author_data, DATA_DIR + 'author_data.json')
def load_dictionary(fpath): try: with open(fpath) as f: data = json.load(f) return data except: print_log('error : could not load file at {}'.format(fpath))
def do_inference(onnx_model): import time num_tests = 100 num_images = 100 total_tests = 0 total_elapsed = 0 #import ipdb as pdb; pdb.set_trace() sess = rt.InferenceSession(onnx_model) input_name = sess.get_inputs()[0].name pass_cnt = 0 utility.print_banner("Starting Inference Engine") for test_id in range(0, num_tests): images, labels = load_images(num_images) inf_start = time.time() pred_onx = sess.run(None, {input_name: images.astype(np.float32)})[0] inf_end = time.time() elapsed_time = inf_end - inf_start total_elapsed += elapsed_time utility.print_log("batch {} took {:2.4f}ms to complete".format( test_id, elapsed_time * 1000)) for i in range(0, num_images): pred = pred_onx[i].argmax() res = "FAIL" if labels[i] == pred: res = "PASS" pass_cnt += 1 total_tests += 1 utility.print_log("actual={} pred={} res={}".format( labels[i], pred, res), verbosity="VERB_HIGH") avg_inf = 1000.0 * float(total_elapsed) / total_tests utility.print_banner( "Accuracy = {}% out of {} tests, avg inference={:2.4}ms per image". format(100 * pass_cnt / (float(num_images) * num_tests), float(num_images) * num_tests, avg_inf))
def save_output(output_vals, output_filename): file_object = open(output_filename, 'w') for i in range(len(output_vals)): temp = "" for j in range(output_vals[i].shape[0]): temp += str(int(output_vals[i][j])) file_object.write(temp + "\n") file_object.close() util.print_log("saving outputs to {0}".format(output_filename))
def convertBin(value): if (value == 1): return "01" elif (value == -1): return "11" elif (value == 0): return "00" else: return "output_size" util.print_log("only ternary weights of -1, 0 and 1 is supported", id_str="ERROR") sys.exit()
def run_model(windows, W1, W2, W3, b1, b2, b3, verbosity="VERB_HIGH"): AC1 = hw.AccelCore(l1_input, l2_input) AC2 = hw.AccelCore(l2_input, l3_input) AC3 = hw.AccelCore(l3_input, output_size) out = [] for i in range(windows.shape[0]): temp_out = AC1.forward(W1, windows[i].flatten(), b1) temp_out = AC2.forward(W2, temp_out, b2) out.append(AC3.forward(W3, temp_out, b3)) if ((i % 100) == 0): util.print_log("Iteration {0}".format(i), verbosity=verbosity) return out
def load_book_data_as_input(folder_path): book_data = load_dictionary(glob.glob(folder_path + 'book_data.json')[0]) dataset= [] ids = [] for key, value in book_data.items(): descr = value['description'] if descr: descr = clean_html(descr.lower()) else: descr = get_name(value['title'].lower()) dataset.append(descr) ids.append(key) print_log("dataset size : {}".format(len(dataset))) return ids, dataset
def book_id2attr_map(folder_path, attributes=['title']): book_data = load_dictionary(glob.glob(folder_path + 'book_data.json')[0]) print_log('total books loaded = {}'.format(len(book_data))) d_list = [dict() for i in range(len(attributes))] for key, value in book_data.items(): for idx, d in enumerate(d_list): if attributes[idx]=='description': descr = value[attributes[idx]] if descr: descr = clean_html(descr.lower()) d[key] = descr else: d[key] = value[attributes[idx]] return d_list
def load_user_data(folder_path, book_id2attr = None): ''' Returns: dataset = list of docs. Each doc is a string. vocabulary = set of unique words ''' all_user_files = glob.glob(folder_path +'user_*.json') dataset = [] for fpath in all_user_files: d = load_dictionary(fpath) books = d.values()[0]['books'].keys() doc = create_user_doc(books, book_id2attr) dataset.append(doc) print_log("dataset size : {}".format(len(dataset))) return dataset
def optimize_model(expr_name, onnx_file, model_xml, weight_bin): run_opt = False if (model_xml == None): util.print_log("Could not find xml model", id_str="warning") run_opt = True if (weight_bin == None): util.print_log("Could not find binary weights", id_str="warning") run_opt = True if run_opt: util.print_banner( "Running OpenVino optimizer on {0}".format(onnx_file), color='green', verbosity="VERB_LOW") cmd = "python {0}/deployment_tools/model_optimizer/mo.py --input_model={1} --model_name {2}".format( openvino_inst_path, onnx_file, expr_name) util.run_command(cmd, verbosity="VERB_LOW") model_xml, weight_bin = expr_name + ".xml", expr_name + ".bin" # load model # import ipdb as pdb; pdb.set_trace() return model_xml, weight_bin
def optimize_model(expr_name, onnx_file, model_xml, weight_bin, opt_ops=""): # import ipdb as pdb; pdb.set_trace() run_opt = False options = "" if (model_xml == None): util.print_log("No xml model was provided", id_str="warning") run_opt = True if (weight_bin == None): util.print_log("No binary weight file was provided", id_str="warning") run_opt = True if run_opt: util.print_banner("Running OpenVino optimizer on {0}".format(onnx_file), color='green', verbosity="VERB_LOW") options += "--input_model={0} ".format(onnx_file) options += "--model_name {0} ".format(expr_name) options += (" --"+opt_ops[0]) if len(opt_ops)==1 else "".join(" --"+e for e in opt_ops) cmd = "python {0}/deployment_tools/model_optimizer/mo_onnx.py {1}".format(openvino_inst_path, options) util.run_command(cmd, verbosity="VERB_LOW") model_xml, weight_bin = expr_name+".xml", expr_name+".bin" # load model # import ipdb as pdb; pdb.set_trace() return model_xml, weight_bin
def reviews(self, page=1): """ Get all books and reviews on user's shelves page = 'all' if all reviews are requested. Note: reviews() function of Goodreads.user class is a bit buggy. If there is only one review to return, it is not handled properly there. """ all_reviews = [] start_page_index = page if page == 'all': start_page_index = 1 while True: try: resp = self._client.session.get("/review/list.xml", { 'v': 2, 'id': self.gid, 'page': start_page_index }) if isinstance(resp['reviews']['review'], list): all_reviews += [ review.GoodreadsReview(r) for r in resp['reviews']['review'] ] else: all_reviews += [ review.GoodreadsReview(resp['reviews']['review']) ] except: break start_page_index += 1 if page != 'all': break print_log('total reviews collected from id={}, name={} :{}'.format( self.gid, get_name(self.name), len(all_reviews))) if len(all_reviews) == 0: print_log("Returning 0 reviews: {}".format(all_reviews)) return all_reviews
def get_rtl_files(f_file): sv_rtl = "" vhdl_rtl = "" v_rtl = "" with open(f_file, 'r') as f: rtls = f.readlines() for rtl in rtls: rtl = rtl.replace("\n", "") if rtl != "": if rtl.lower().endswith(".vhdl") or rtl.lower().endswith( ".vhd"): vhdl_rtl += "{0} ".format(rtl) elif rtl.lower().endswith(".sv") or rtl.lower().endswith( ".svh"): sv_rtl += "{0} ".format(rtl) elif rtl.lower().endswith(".v") or rtl.lower().endswith(".vh"): v_rtl += "{0} ".format(rtl) else: util.print_log("unsupported file format: {0}".format(rtl), "ERROR", verbosity="VERB_LOW") sys.exit() # import ipdb as pdb; pdb.set_trace() return sv_rtl, v_rtl, vhdl_rtl
w_cnt += 1 return window if __name__ == '__main__': args = parse_args() verbosity = args['verbosity'] verbosity = verbosity if verbosity != None else "VERB_HIGH" input_img_width = int(args['input_img_width']) input_img_height = int(args['input_img_height']) kernel_size = int(args['kernel_size']) input_img = open(args['input_img'], "r").read() output_window = open(args['output_window'], "r").read() util.print_banner("Testing Window Slider Output ", verbosity='VERB_LOW') util.print_log("input image: {0} ".format(args['input_img'])) util.print_log("simulation output: {0} ".format(args['output_window'])) input_img = input_img.replace("\n", ",").split(",") input_img = input_img[:-1] input_img = [int(x) for x in input_img] input_img = np.asarray(input_img) input_img = input_img.reshape((input_img_height, input_img_width)) sw_output = gen_window(image=input_img, w_height=kernel_size, w_width=kernel_size, stride=1) output_window = output_window.replace("1'b", "").replace("'{", "").replace( "}", "").split("\n") total_num_windows = ((input_img_height - kernel_size) + 1) * ( (input_img_width - kernel_size) + 1)
gui = args['gui'] svseed = args['svseed'] coverage = args['coverage'] debug = args['debug'] waveform = args['waveform'] clean = args['clean'] silence = args['silence'] verbosity = args['verbosity'] compileonly = args['compileonly'] elabonly = args['elabonly'] if verbosity is None: verbosity = 'VERB_LOW' if util.get_platform(verbosity=verbosity) != "linux": util.print_log("This script works only on a Linux platform", "ERROR", verbosity="VERB_LOW") sys.exit() if clean: util.print_banner("Cleaning project", verbosity=verbosity) util.clean_proj(files_to_clean) if not os.path.exists(result_dir): util.print_log("Creating a result directory in {0}".format(result_dir), "INFO", verbosity="VERB_LOW") os.makedirs(result_dir) if simulator == None: util.print_log("You need to provide Simulator name", "ERROR", verbosity="VERB_LOW") sys.exit()
def get_vocabulary(dataset): vocabulary = list(set(" ".join(dataset).split())) print_log("vocabulary size : {}".format(len(vocabulary))) return vocabulary
def load_params(img_npy, w1_npy, w2_npy, w3_npy, b1_npy, b2_npy, b3_npy, w_width, w_height, stride): # import ipdb as pdb; pdb.set_trace() util.print_log("loading {0}...".format(img_npy)) image = np.load(img_npy) windows = (gen_window(image, w_width, w_height, stride)) util.print_log("loading {0}...".format(w1_npy)) w1 = np.load(w1_npy) util.print_log("loading {0}...".format(w2_npy)) w2 = np.load(w2_npy) util.print_log("loading {0}...".format(w3_npy)) w3 = np.load(w3_npy) util.print_log("loading {0}...".format(b1_npy)) b1 = np.load(b1_npy) util.print_log("loading {0}...".format(b2_npy)) b2 = np.load(b2_npy) util.print_log("loading {0}...".format(b3_npy)) b3 = np.load(b3_npy) return windows, w1, w2, w3, b1, b2, b3
#======================================================================= # Main #======================================================================= if __name__ == '__main__': cmd_to_run = "" __args__ = parse_args() project_name = __args__['project_name'] path_for_proj = __args__['path_proj'] path_fpga = __args__['path_fpga'] verbosity = __args__['vebosity'] util.print_banner("Creating {0} Project".format(project_name), verbosity="VERB_LOW") # import ipdb as pdb; pdb.set_trace() if path_for_proj == None: default_path_for_proj = os.getcwd() + "/" util.print_log("Using current path for creating project: {0}".format( default_path_for_proj)) path_for_proj = default_path_for_proj if path_fpga == None: util.print_log("Using default path to fpga directory {0}".format( default_path_to_fpga_dir)) path_fpga = default_path_to_fpga_dir # Check if project has already been created if os.path.isdir("{0}{1}".format(path_for_proj, project_name)): util.print_log("Project path {0}{1} already exist!".format( path_for_proj, project_name), id_str="ERROR") sys.exit() if os.getcwd() not in path_for_proj: proj_dir = os.getcwd() + "/" + path_for_proj + project_name else: proj_dir = path_for_proj + project_name
for author in book.authors: if not author_data.has_key(author.gid): author_data[author.gid] = extract_author_info(author) except: print_log("error : review for friend_id={} review_dict {}".format(friend.gid, review._review_dict)) friend_name = '_'.join(get_name(friend.name).split()).lower() fname = DATA_DIR + 'user_{}_{}.json'.format(friend_name, friend.gid) save_dictionary(user_data, fname) save_dictionary(book_data, DATA_DIR + 'book_data.json') save_dictionary(author_data, DATA_DIR + 'author_data.json') if __name__ == "__main__": logging.basicConfig(filename='result.log', level=logging.INFO) gc = client.GoodreadsClient(APP_KEY, APP_SECRET) #print_log(gc.session.access_token) #print_log(gc.session.access_token_secret) gc.authenticate(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) user = gc.auth_user() print_log("Hello I am {} with id = {}".format(user.name, user.gid)) user = GoodreadsUserExtended.fromBaseUser(user) friend_list = user.friends(all=True) all_ids = [u.gid for u in friend_list+[user]] collect_reviews(gc, all_ids) print_log('total books = {}'.format(len(book_data))) print_log('total authors = {}'.format(len(author_data))) print_log('total users = {}'.format(len(all_ids))) exit()
def get_reviews(gc, user_id='29400625'): user = GoodreadsUserExtended.fromBaseUser(gc.user(user_id)) reviews = user.reviews() for review in reviews: print_log(easy_json(review.book)) return
def save_dictionary(data, fpath): with open(fpath, 'w') as f: json.dump(data, f) print_log("Saved as {}".format(fpath))
top_level = args['top_level'] files = args['files'] gui = args['gui'] svseed = args['svseed'] coverage = args['coverage'] debug = args['debug'] waveform = args['waveform'] clean = args['clean'] # import ipdb as pdb; pdb.set_trace() silence = args['silence'] verbosity = args['verbosity'] if verbosity is None: verbosity = 'VERB_LOW' if util.get_platform(verbosity=verbosity) != "linux": util.print_log("This script works only on a Linux platform", "ERROR", verbosity="VERB_LOW") sys.exit() if clean: util.print_banner("Cleaning project", verbosity=verbosity) util.clean_proj(files_to_clean) if not os.path.exists(result_dir): util.print_log("Creating a result directory in {0}".format(result_dir), "INFO", verbosity="VERB_LOW") os.makedirs(result_dir) if simulator == None: util.print_log("You need to provide Simulator name", "ERROR", verbosity="VERB_LOW") sys.exit()
get_name(d.values()[0]['name']),\ d.values()[0]['books'].keys() #TODO: fetch indices of books user has read. if __name__ == "__main__": logging.basicConfig(filename='result.log', filemode='w',level=logging.INFO) n_topics = 10 n_top_words = 10 n_features = 1000 min_df = 2 max_df = 0.2 max_iter = 10 print_log('n_topics = {}, n_top_words = {}'.format(n_topics, n_top_words)) print_log('min_df = {}, max_df = {}'.format(min_df, max_df)) book_id2title, book_id2description = book_id2attr_map(DATA_DIR, attributes=['title','description']) # dataset = load_user_data(DATA_DIR, book_id2description) book_ids, dataset = load_book_data_as_input(DATA_DIR) vocabulary = get_vocabulary(dataset) # print_log(book_id2description['1362']) print("Extracting tf features for LDA...") tf_vectorizer = CountVectorizer(max_df=max_df, min_df=min_df, max_features=n_features, #vocabulary=vocabulary, stop_words='english') t0 = time() tf = tf_vectorizer.fit_transform(dataset)
def print_top_words(model, feature_names, n_top_words, book_data = None): for topic_idx, topic in enumerate(model.components_): print_log("Topic #%d:" % topic_idx) print_log(", ".join([get_name(feature_names[i]) for i in topic.argsort()[:-n_top_words - 1:-1]]))
util.print_log("saving outputs to {0}".format(output_filename)) if __name__ == '__main__': args = parse_args() output_filename = args['output_filename'] operation_type = args['operation_type'] verbosity = args['verbosity'] if output_filename == None: output_filename = "output.txt" if operation_type == None: operation_type = "re_run" if verbosity == None: verbosity = "VERB_HIGH" util.print_banner("Running HW simulator", verbosity="VERB_LOW") if operation_type.lower() == "re_run": util.print_log("re-running the model...", verbosity="VERB_LOW") windows, W1, W2, W3, b1, b2, b3 = hw.load_params( "image.npy", "w1.npy", "w2.npy", "w3.npy", "b1.npy", "b2.npy", "b3.npy", w_width, w_height, stride) elif operation_type.lower() == "normal": util.print_log("running in normal model...", verbosity="VERB_LOW") windows, W1, W2, W3, b1, b2, b3 = hw.gen_random_params( input_file_name, image_h, image_w, w_width, w_height, stride, l1_input, l2_input, l3_input, output_size) hw.save_model(W1, W2, W3) hw.gen_sv_weight_file("weight.svh", W1, W2, W3, b1, b2, b3) out = run_model(windows, W1, W2, W3, b1, b2, b3, verbosity) save_output(out, output_filename)